From 7b4c78bbffab8242fdaa90cea4c2c45b6e401a13 Mon Sep 17 00:00:00 2001 From: Florian Habermann Date: Tue, 18 Mar 2025 14:52:18 +0100 Subject: [PATCH 01/14] Update version to 2.1.3 across documentation (#391) Bumped the display, Maven, and tooling versions from 2.1.2 to 2.1.3 in Antora configuration and various README files. Added a changelog entry for 2.1.3 outlining key changes, including new features, refactors, and removals. --- docs/antora.yml | 4 ++-- docs/modules/intro/pages/changelog.adoc | 10 ++++++++++ storage/embedded-tools/storage-converter/README.md | 2 +- storage/embedded-tools/storage-migrator/README.md | 6 +++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/antora.yml b/docs/antora.yml index edccca3f..d09780a4 100644 --- a/docs/antora.yml +++ b/docs/antora.yml @@ -8,6 +8,6 @@ nav: asciidoc: attributes: product-name: 'EclipseStore' - display-version: '2.1.2' + display-version: '2.1.3' api-version: '2' - maven-version: '2.1.2' + maven-version: '2.1.3' diff --git a/docs/modules/intro/pages/changelog.adoc b/docs/modules/intro/pages/changelog.adoc index f7d886d4..f6a96728 100644 --- a/docs/modules/intro/pages/changelog.adoc +++ b/docs/modules/intro/pages/changelog.adoc @@ -1,6 +1,16 @@ = Changelog +== 2.1.3 + +=== Changes + +* Add viewRoots() to PersistenceManager https://github.com/eclipse-serializer/serializer/pull/178[[178]] +* Add referential equality to Equalator https://github.com/eclipse-serializer/serializer/pull/184[[184]] +* Remove unused and deprecated stuff https://github.com/eclipse-serializer/serializer/pull/179[[179]] https://github.com/eclipse-serializer/serializer/pull/180[[180]] +* Refactor CacheStore and improve backend code quality https://github.com/eclipse-store/store/pull/390[[390]] + + == 2.1.2 === Bugfixes diff --git a/storage/embedded-tools/storage-converter/README.md b/storage/embedded-tools/storage-converter/README.md index 217dde0b..8e21d88d 100644 --- a/storage/embedded-tools/storage-converter/README.md +++ b/storage/embedded-tools/storage-converter/README.md @@ -22,7 +22,7 @@ mvn -Pconverter-standalone clean package To configure the input and output storage an [external configuration](https://docs.eclipsestore.io/manual/storage/configuration/index.html#external-configuration) file for each storage is required. ```console -java -jar storage-embedded-tools-storage-converter-2.1.2.jar sourceCongig.xml targetConfig.xml +java -jar storage-embedded-tools-storage-converter-2.1.3.jar sourceCongig.xml targetConfig.xml ``` ### StorageConverter.java diff --git a/storage/embedded-tools/storage-migrator/README.md b/storage/embedded-tools/storage-migrator/README.md index ec9b89e2..d77fda5f 100644 --- a/storage/embedded-tools/storage-migrator/README.md +++ b/storage/embedded-tools/storage-migrator/README.md @@ -16,17 +16,17 @@ The type dictionary migration is only done when `typeDictionaryRelativePath` is ### Migration of both, source code and type dictionary: ```` -mvn org.openrewrite.maven:rewrite-maven-plugin:run -Drewrite.activeRecipes=org.eclipse.store.storage.embedded.tools.storage.migrator.ConvertProject -Drewrite.recipeArtifactCoordinates=org.eclipse.store:storage-embedded-tools-storage-migrator:2.1.2 -DeclipseStoreVersion=2.1.2 -Drewrite.plainTextMasks=**/*.ptd -DtypeDictionaryRelativeFilePath=src/main/resources/PersistenceTypeDictionary.ptd +mvn org.openrewrite.maven:rewrite-maven-plugin:run -Drewrite.activeRecipes=org.eclipse.store.storage.embedded.tools.storage.migrator.ConvertProject -Drewrite.recipeArtifactCoordinates=org.eclipse.store:storage-embedded-tools-storage-migrator:2.1.3 -DeclipseStoreVersion=2.1.3 -Drewrite.plainTextMasks=**/*.ptd -DtypeDictionaryRelativeFilePath=src/main/resources/PersistenceTypeDictionary.ptd ```` ### Migration of source code only: ```` -mvn org.openrewrite.maven:rewrite-maven-plugin:run -Drewrite.activeRecipes=org.eclipse.store.storage.embedded.tools.storage.migrator.ConvertProject -Drewrite.recipeArtifactCoordinates=org.eclipse.store:storage-embedded-tools-storage-migrator:2.1.2 -DeclipseStoreVersion=2.1.2 +mvn org.openrewrite.maven:rewrite-maven-plugin:run -Drewrite.activeRecipes=org.eclipse.store.storage.embedded.tools.storage.migrator.ConvertProject -Drewrite.recipeArtifactCoordinates=org.eclipse.store:storage-embedded-tools-storage-migrator:2.1.3 -DeclipseStoreVersion=2.1.3 ```` ### Migration of type dictionary only: ```` -mvn org.openrewrite.maven:rewrite-maven-plugin:run -Drewrite.activeRecipes=org.eclipse.store.storage.embedded.tools.storage.migrator.ConvertProject -Drewrite.recipeArtifactCoordinates=org.eclipse.store:storage-embedded-tools-storage-migrator:2.1.2 -Drewrite.plainTextMasks=**/*.ptd -DtypeDictionaryRelativeFilePath=src/main/resources/PersistenceTypeDictionary.ptd +mvn org.openrewrite.maven:rewrite-maven-plugin:run -Drewrite.activeRecipes=org.eclipse.store.storage.embedded.tools.storage.migrator.ConvertProject -Drewrite.recipeArtifactCoordinates=org.eclipse.store:storage-embedded-tools-storage-migrator:2.1.3 -Drewrite.plainTextMasks=**/*.ptd -DtypeDictionaryRelativeFilePath=src/main/resources/PersistenceTypeDictionary.ptd ```` From baf494bb1196f390b1e76e11b8cee509ef2e8227 Mon Sep 17 00:00:00 2001 From: Florian Habermann Date: Tue, 1 Apr 2025 11:54:08 +0200 Subject: [PATCH 02/14] Remove unnecessary synchronization from housekeeping methods (#395) The `reset` and `increaseNs` methods no longer require the `synchronized` keyword, as their operations are either inherently thread-safe or already guarded externally. This change simplifies the code and may improve performance by reducing locking overhead. --- .../store/storage/types/StorageHousekeepingController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageHousekeepingController.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageHousekeepingController.java index 296659e3..f1646d43 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageHousekeepingController.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageHousekeepingController.java @@ -613,7 +613,7 @@ public static long defaultAdaptiveHousekeepingMaximumTimeBudgetNs() // methods // //////////// - private synchronized void reset() + private void reset() { this.lastFinishedGCCycle = this.lastIncrease = System.currentTimeMillis(); if(this.currentIncreaseNs != 0) @@ -623,7 +623,7 @@ private synchronized void reset() } } - private synchronized long increaseNs() + private long increaseNs() { if(this.monitorSupplier.get().isComplete()) { From efc6008228c50c2033eff02d7a91412175e90bd1 Mon Sep 17 00:00:00 2001 From: Zdenek Jonas Date: Wed, 30 Apr 2025 09:07:20 +0200 Subject: [PATCH 03/14] chore: update Spring Boot version to 3.4.5 in pom.xml (#403) --- examples/spring-boot3-advanced/pom.xml | 2 +- examples/spring-boot3-simple/pom.xml | 2 +- integrations/spring-boot3-console/pom.xml | 2 +- integrations/spring-boot3/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/spring-boot3-advanced/pom.xml b/examples/spring-boot3-advanced/pom.xml index d5547dc7..81541e16 100644 --- a/examples/spring-boot3-advanced/pom.xml +++ b/examples/spring-boot3-advanced/pom.xml @@ -18,7 +18,7 @@ 17 17 UTF-8 - 3.2.2 + 3.4.5 diff --git a/examples/spring-boot3-simple/pom.xml b/examples/spring-boot3-simple/pom.xml index 6cddce4c..9986edb0 100644 --- a/examples/spring-boot3-simple/pom.xml +++ b/examples/spring-boot3-simple/pom.xml @@ -18,7 +18,7 @@ 17 17 UTF-8 - 3.2.2 + 3.4.5 diff --git a/integrations/spring-boot3-console/pom.xml b/integrations/spring-boot3-console/pom.xml index f1e25a36..4cb3fe32 100644 --- a/integrations/spring-boot3-console/pom.xml +++ b/integrations/spring-boot3-console/pom.xml @@ -16,7 +16,7 @@ https://projects.eclipse.org/projects/technology.store - 3.2.0 + 3.4.5 diff --git a/integrations/spring-boot3/pom.xml b/integrations/spring-boot3/pom.xml index fe800aab..fa15453d 100644 --- a/integrations/spring-boot3/pom.xml +++ b/integrations/spring-boot3/pom.xml @@ -16,7 +16,7 @@ https://projects.eclipse.org/projects/technology.store - 3.2.2 + 3.4.5 From 437296d7311d6112a8add5c914410d40eb80f9df Mon Sep 17 00:00:00 2001 From: Florian Habermann Date: Wed, 30 Apr 2025 10:39:29 +0200 Subject: [PATCH 04/14] Enhance storage system interfaces with detailed JavaDocs (#402) Added JavaDocs to `StorageSystem`, `StorageController`, and `StorageActivePart` interfaces to clarify their functionalities and usage. --- .../storage/types/StorageActivePart.java | 9 ++ .../storage/types/StorageController.java | 101 ++++++++++++++++-- .../store/storage/types/StorageSystem.java | 88 +++++++++++++-- 3 files changed, 186 insertions(+), 12 deletions(-) diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageActivePart.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageActivePart.java index 69d5dfd8..1bccec41 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageActivePart.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageActivePart.java @@ -14,6 +14,15 @@ * #L% */ + +/** + * Interface representing an active component of a storage system. + *

+ * Implementors of this interface represent parts of a storage system + * that can be queried for their active status. The active state typically + * indicates whether the part is currently operational or engaged in its + * designated tasks. + */ public interface StorageActivePart { /** diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageController.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageController.java index 949e01b4..9129a8bd 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageController.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageController.java @@ -16,6 +16,15 @@ import org.eclipse.store.storage.exceptions.StorageExceptionShutdown; + +/** + * The StorageController interface defines the contract for managing and controlling a storage system. + * It provides methods to initialize, start, monitor, and shut down the storage operations, ensuring + * appropriate handling of state transitions and task processing capabilities. + * + * This interface extends both StorageActivePart and AutoCloseable, combining functionalities for + * active storage management and resource cleanup during lifecycle management. + */ public interface StorageController extends StorageActivePart, AutoCloseable { /** @@ -36,31 +45,111 @@ public interface StorageController extends StorageActivePart, AutoCloseable * if an internal {@link InterruptedException} happened. */ public boolean shutdown(); - + + /** + * Checks whether the storage controlled by this instance is currently accepting tasks. + * This indicates the readiness of the storage to handle and process new tasks, + * based on its internal state and operational status. + * + * @return true if the storage is currently accepting tasks, false otherwise. + */ public boolean isAcceptingTasks(); - + + /** + * Checks whether the storage controlled by this instance is currently running. + * This method determines if the storage is operational and handling tasks + * after being successfully started and not yet shut down. + * + * @return true if the storage is running, false otherwise. + */ public boolean isRunning(); - + + /** + * Determines whether the storage controlled by this instance is currently in the process + * of starting up. This state indicates that the storage is initializing, which includes + * activities such as reading and indexing data, initializing resources, or starting + * background management processes. + * + * @return true if the storage is in the process of starting up, false otherwise. + */ public boolean isStartingUp(); - + + /** + * Determines whether this instance is currently in the process of shutting down. + * + * @return true if the storage is in the process of shutting down, false otherwise. + */ public boolean isShuttingDown(); - + + /** + * Checks whether the storage controlled by this instance has been shut down. + * + * This method determines the shutdown state by checking if the storage is no longer running. + * + * @return true if the storage is shut down, false otherwise. + */ public default boolean isShutdown() { return !this.isRunning(); } - + + /** + * Ensures that the storage is accepting tasks by internally checking its state. + * This method verifies the ability of the storage to process new tasks and may + * throw an exception if the storage is not in a state to accept tasks. + * + * Depending on the internal implementation, it could check conditions like whether + * the storage is running, not shutting down, and has not encountered critical errors. + * + * @throws IllegalStateException if the storage is not currently accepting tasks. + */ public void checkAcceptingTasks(); + /** + * Retrieves the timestamp that represents the initialization time of the storage. + * This method provides the time when the initialization process was completed successfully. + * + * @return the initialization time as a long value representing the timestamp + * in milliseconds since the epoch (January 1, 1970, 00:00:00 GMT). + */ public long initializationTime(); + /** + * Retrieves the timestamp that represents the time when the storage + * entered its current operational mode. This method provides a long + * value indicating the moment when the storage transitioned into its + * active state. + * + * @return the timestamp as a long value representing the time in milliseconds + * since the epoch (January 1, 1970, 00:00:00 GMT) when the storage + * entered its current operational mode. + */ public long operationModeTime(); + /** + * Computes the duration of the initialization process for the storage. + * This value represents the time elapsed between the completion of the + * initialization process and the moment the storage entered its + * operational mode. + * + * @return the initialization duration as a long value representing the + * time in milliseconds. + */ public default long initializationDuration() { return this.operationModeTime() - this.initializationTime(); } + /** + * Closes the storage controller by shutting down all associated resources and + * threads managing the storage. This method ensures that the storage is properly + * terminated and no background processes are left running. + * + * If the shutdown process fails, a {@link StorageExceptionShutdown} is thrown with + * an appropriate error message and cause if available. + * + * @throws StorageExceptionShutdown if the shutdown process is unsuccessful. + */ @Override public default void close() throws StorageExceptionShutdown { diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageSystem.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageSystem.java index 7e95f2ee..5dbe98c0 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageSystem.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageSystem.java @@ -33,23 +33,83 @@ import org.eclipse.store.storage.exceptions.StorageExceptionNotRunning; import org.slf4j.Logger; -// (21.03.2016 TM)TODO: what is the difference between ~Manager and ~Controller here? Merge into Controller or comment. + +/** + * The StorageSystem class represents the core of a storage system implementation, providing + * key functionalities to manage, configure, and interact with a storage setup. It encapsulates + * various components that contribute to the operation, monitoring, and configuration of the + * storage system. + *

+ * Core functionalities include + *

    + *
  • Managing storage requests, operations, and configurations
    • + *
  • Monitoring object ID ranges and initialization processes
    • + *
  • Providing access to type dictionaries and file systems
    • + *
  • Handling startup and shutdown operations of the storage system
    • + *
+ */ public interface StorageSystem extends StorageController { + /** + * Creates and returns an instance of {@link StorageRequestAcceptor}, which is responsible for handling + * various storage-related tasks, such as storing data, querying data, managing garbage collection, + * and performing other storage operations. + * + * @return an instance of {@link StorageRequestAcceptor} that facilitates interaction with the storage system. + */ public StorageRequestAcceptor createRequestAcceptor(); - + + /** + * Provides access to the storage type dictionary, which is used to manage and interact + * with definitions and handlers for different storage entity types within the system. + * + * @return an instance of {@link StorageTypeDictionary}, which contains type definitions + * and provides methods to validate, initialize, and look up type handlers and definitions. + */ public StorageTypeDictionary typeDictionary(); - - // (20.05.2013 TM)TODO: StorageManager#operationController() - not sure this belongs here + + /** + * Provides an instance of {@link StorageOperationController}, which manages + * the state and operations of the storage system, including channel + * processing and disruption handling. + * + * @return an instance of {@link StorageOperationController} for controlling + * storage operations. + */ public StorageOperationController operationController(); + /** + * Provides an instance of {@link StorageChannelCountProvider}, which determines + * the number of channels available for storage operations in the system. + * The returned provider can be used to retrieve channel count configurations + * and manage parallel storage operations efficiently. + * + * @return an instance of {@link StorageChannelCountProvider} that supplies + * channel count information to optimize storage processes. + */ public default StorageChannelCountProvider channelCountProvider() { return this.operationController().channelCountProvider(); } - + + /** + * Provides the configuration details for the storage system. This method returns an instance of + * {@link StorageConfiguration}, which contains various parameters and settings that dictate how + * the storage system should operate, including performance tuning, caching policies, and storage + * allocation strategies. + * + * @return an instance of {@link StorageConfiguration} that encapsulates the configuration parameters + * and operational settings for the storage system. + */ public StorageConfiguration configuration(); + /** + * Provides access to the underlying {@link AFileSystem} instance associated with the storage system. + * The file system is obtained via the configuration's file provider and enables low-level file + * operations and management within the storage system. + * + * @return an instance of {@link AFileSystem} representing the file system used by the storage system. + */ public default AFileSystem fileSystem() { return this.configuration().fileProvider().fileSystem(); @@ -59,11 +119,27 @@ public default AFileSystem fileSystem() @Override public StorageSystem start(); + /** + * Initializes and provides an instance of {@link StorageIdAnalysis}, which is responsible + * for analyzing and maintaining information about unique identifiers within the storage + * system, such as the highest IDs per type and the IDs that are currently in use. + * + * @return an instance of {@link StorageIdAnalysis}, which provides methods to retrieve + * information about identifier usage and analysis in the storage system. + */ public StorageIdAnalysis initializationIdAnalysis(); @Override public boolean shutdown(); - + + /** + * Provides an instance of {@link StorageObjectIdRangeEvaluator}, which is responsible + * for evaluating ranges of object IDs within the storage system. This evaluation can be + * useful for validating or processing object ID ranges to manage the storage system's state + * or integrity. + * + * @return an instance of {@link StorageObjectIdRangeEvaluator} used for evaluating object ID ranges. + */ public StorageObjectIdRangeEvaluator objectIdRangeEvaluator(); From 48f62479d554dc7902d084dd6d14c54d7d5d3ab3 Mon Sep 17 00:00:00 2001 From: Zdenek Jonas Date: Mon, 19 May 2025 08:57:35 +0200 Subject: [PATCH 05/14] chore: update jetty.version to 9.4.57.v20241219 in pom.xml (#404) --- storage/rest/service-sparkjava/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/rest/service-sparkjava/pom.xml b/storage/rest/service-sparkjava/pom.xml index 2854ebf2..1f379360 100644 --- a/storage/rest/service-sparkjava/pom.xml +++ b/storage/rest/service-sparkjava/pom.xml @@ -21,7 +21,7 @@ 2.9.3 - 9.4.56.v20240826 + 9.4.57.v20241219 2.8.9 From 6b0fc632be7044954128f13a9ae9056f693d0f13 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Tue, 27 May 2025 14:30:43 +0200 Subject: [PATCH 06/14] added public long store(Object instance, long objectId) (#406) --- .../examples/extensionwrapper/PersistenceStorerWrapper.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/extension-wrapper/src/main/java/org/eclipse/store/examples/extensionwrapper/PersistenceStorerWrapper.java b/examples/extension-wrapper/src/main/java/org/eclipse/store/examples/extensionwrapper/PersistenceStorerWrapper.java index d7a5956c..2085449d 100644 --- a/examples/extension-wrapper/src/main/java/org/eclipse/store/examples/extensionwrapper/PersistenceStorerWrapper.java +++ b/examples/extension-wrapper/src/main/java/org/eclipse/store/examples/extensionwrapper/PersistenceStorerWrapper.java @@ -133,6 +133,12 @@ public void registerRegistrationListener(PersistenceObjectRegistrationListener l { this.delegate.registerRegistrationListener(listener); } + + @Override + public long store(Object instance, long objectId) + { + return this.delegate.store(instance, objectId); + } } From 26ddbc4257cfdc2ee449562abc001b265a9a89d4 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Wed, 28 May 2025 06:36:12 +0200 Subject: [PATCH 07/14] Feature/storage entity collector creator allow storage start with missing objects (#407) StorageFoundation can be configured to use a StorageEntityCollector that does not throw an execption on missing objects: ``` storageFoundation.setStorageEntityCollectorCreator(StorageEntityCollector.Creator.Unchecked()) ``` --- .../store/storage/types/StorageChannel.java | 165 +++--------- .../storage/types/StorageChannelsCreator.java | 9 +- .../storage/types/StorageEntityCollector.java | 236 ++++++++++++++++++ .../storage/types/StorageFoundation.java | 55 +++- .../store/storage/types/StorageSystem.java | 8 +- 5 files changed, 327 insertions(+), 146 deletions(-) create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/types/StorageEntityCollector.java diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannel.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannel.java index b1d99a6d..9d4dfb45 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannel.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannel.java @@ -25,7 +25,6 @@ import org.eclipse.serializer.afs.types.AWritableFile; import org.eclipse.serializer.collections.BulkList; import org.eclipse.serializer.functional.ThrowingProcedure; -import org.eclipse.serializer.functional._longProcedure; import org.eclipse.serializer.monitoring.MonitoringManager; import org.eclipse.serializer.persistence.binary.types.Chunk; import org.eclipse.serializer.persistence.binary.types.ChunksBuffer; @@ -38,7 +37,6 @@ import org.eclipse.serializer.util.BufferSizeProviderIncremental; import org.eclipse.serializer.util.X; import org.eclipse.serializer.util.logging.Logging; -import org.eclipse.store.storage.exceptions.StorageExceptionConsistency; import org.eclipse.store.storage.monitoring.StorageChannelHousekeepingMonitor; import org.slf4j.Logger; @@ -117,17 +115,18 @@ public final class Default implements StorageChannel, Unpersistable, StorageHous // instance fields // //////////////////// - private final int channelIndex ; - private final StorageExceptionHandler exceptionHandler ; - private final StorageTaskBroker taskBroker ; - private final StorageOperationController operationController ; - private final StorageHousekeepingController housekeepingController ; - private final StorageHousekeepingBroker housekeepingBroker ; - private final StorageFileManager.Default fileManager ; - private final StorageEntityCache.Default entityCache ; - private final boolean switchByteOrder ; - private final BufferSizeProviderIncremental loadingBufferSizeProvider; - private final StorageEventLogger eventLogger ; + private final int channelIndex ; + private final StorageExceptionHandler exceptionHandler ; + private final StorageTaskBroker taskBroker ; + private final StorageOperationController operationController ; + private final StorageHousekeepingController housekeepingController ; + private final StorageHousekeepingBroker housekeepingBroker ; + private final StorageFileManager.Default fileManager ; + private final StorageEntityCache.Default entityCache ; + private final boolean switchByteOrder ; + private final BufferSizeProviderIncremental loadingBufferSizeProvider; + private final StorageEventLogger eventLogger ; + private final StorageEntityCollector.Creator entityCollectorCreator ; private final HousekeepingTask[] housekeepingTasks; @@ -155,18 +154,19 @@ public final class Default implements StorageChannel, Unpersistable, StorageHous ///////////////// public Default( - final int hashIndex , - final StorageExceptionHandler exceptionHandler , - final StorageTaskBroker taskBroker , - final StorageOperationController operationController , - final StorageHousekeepingBroker housekeepingBroker , - final StorageHousekeepingController housekeepingController , - final StorageEntityCache.Default entityCache , - final boolean switchByteOrder , - final BufferSizeProviderIncremental loadingBufferSizeProvider, - final StorageFileManager.Default fileManager , - final StorageEventLogger eventLogger , - final MonitoringManager monitorManager + final int hashIndex , + final StorageExceptionHandler exceptionHandler , + final StorageTaskBroker taskBroker , + final StorageOperationController operationController , + final StorageHousekeepingBroker housekeepingBroker , + final StorageHousekeepingController housekeepingController , + final StorageEntityCache.Default entityCache , + final boolean switchByteOrder , + final BufferSizeProviderIncremental loadingBufferSizeProvider, + final StorageFileManager.Default fileManager , + final StorageEventLogger eventLogger , + final MonitoringManager monitorManager , + final StorageEntityCollector.Creator entityCollectorCreator ) { super(); @@ -181,6 +181,7 @@ public Default( this.loadingBufferSizeProvider = notNull(loadingBufferSizeProvider); this.eventLogger = notNull(eventLogger) ; this.switchByteOrder = switchByteOrder ; + this.entityCollectorCreator = notNull(entityCollectorCreator) ; // depends on this.fileManager! this.housekeepingTasks = this.defineHouseKeepingTasks(); @@ -632,7 +633,7 @@ public final ChunksBuffer collectLoadByOids(final ChunksBuffer[] resultArray, fi if(!loadOids.isEmpty()) { // progress must have been incremented accordingly at task creation time - loadOids.iterate(new EntityCollectorByOid(this.entityCache, chunks)); + loadOids.iterate(this.entityCollectorCreator.create(this.entityCache, chunks)); } return chunks.complete(); @@ -654,7 +655,7 @@ public final ChunksBuffer collectLoadByTids(final ChunksBuffer[] resultArray, fi if(!loadTids.isEmpty()) { // progress must have been incremented accordingly at task creation time - loadTids.iterate(new EntityCollectorByTid(this.entityCache, chunks)); + loadTids.iterate(new StorageEntityCollector.EntityCollectorByTid(this.entityCache, chunks)); } return chunks.complete(); } @@ -817,116 +818,6 @@ public final void dispose() } } - - - public final class EntityCollectorByOid implements _longProcedure - { - // (01.06.2013 TM)TODO: clean up / consolidate all internal implementations - - /////////////////////////////////////////////////////////////////////////// - // instance fields // - //////////////////// - - private final StorageEntityCache.Default entityCache ; - private final ChunksBuffer dataCollector; - - - - /////////////////////////////////////////////////////////////////////////// - // constructors // - ///////////////// - - public EntityCollectorByOid( - final StorageEntityCache.Default entityCache , - final ChunksBuffer dataCollector - ) - { - super(); - this.entityCache = entityCache ; - this.dataCollector = dataCollector; - } - - - - /////////////////////////////////////////////////////////////////////////// - // methods // - //////////// - - @Override - public final void accept(final long objectId) - { - final StorageEntity.Default entry; - if((entry = this.entityCache.getEntry(objectId)) == null) - { - /* (14.01.2015 TM)NOTE: this actually is an error, as every oid request comes - * from a referencing entity from inside the same database. So if any load request lookup - * yields null, it is an inconsistency that has to be expressed rather sooner than later. - * - * If some kind of querying request (look if an arbitrary oid yields an entity) is needed, - * is has to be a dedicated kind of request, not this one. - * This one does recursive graph loading (consistency required), not arbitrary querying - * with optional results. - */ - throw new StorageExceptionConsistency("No entity found for objectId " + objectId); - } - entry.copyCachedData(this.dataCollector); - this.entityCache.checkForCacheClear(entry, System.currentTimeMillis()); - } - - } - - public final class EntityCollectorByTid implements _longProcedure - { - /////////////////////////////////////////////////////////////////////////// - // instance fields // - //////////////////// - - private final StorageEntityCache.Default entityCache ; - private final ChunksBuffer dataCollector; - - - - /////////////////////////////////////////////////////////////////////////// - // constructors // - ///////////////// - - public EntityCollectorByTid( - final StorageEntityCache.Default entityCache , - final ChunksBuffer dataCollector - ) - { - super(); - this.entityCache = entityCache ; - this.dataCollector = dataCollector; - } - - - - /////////////////////////////////////////////////////////////////////////// - // methods // - //////////// - - @Override - public final void accept(final long tid) - { - final StorageEntityType.Default type; - if((type = this.entityCache.getType(tid)) == null) - { - // it can very well be that a channel does not have a certain type at all. That is no error - return; - } - - // all the type's entities are iterated and their data is collected - for(StorageEntity.Default entity = type.head; (entity = entity.typeNext) != null;) - { - entity.copyCachedData(this.dataCollector); - this.entityCache.checkForCacheClear(entity, System.currentTimeMillis()); - } - } - - } - - @FunctionalInterface public interface HousekeepingTask { diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannelsCreator.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannelsCreator.java index a5a4cb2a..c8bd81ba 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannelsCreator.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannelsCreator.java @@ -52,7 +52,8 @@ public StorageChannel[] createChannels( Referencing refStorerRegistry , boolean switchByteOrder , long rootTypeId , - MonitoringManager monitorManager + MonitoringManager monitorManager , + StorageEntityCollector.Creator entityCollectorCreator ); @@ -89,7 +90,8 @@ public final StorageChannel.Default[] createChannels( final Referencing refStorerRegistry , final boolean switchByteOrder , final long rootTypeId , - final MonitoringManager monitorManager + final MonitoringManager monitorManager , + final StorageEntityCollector.Creator entityCollectorCreator ) { // (14.07.2016 TM)TODO: make configuration dynamic @@ -168,7 +170,8 @@ public final StorageChannel.Default[] createChannels( loadingBufferSizeProvider, fileManager , eventLogger , - monitorManager + monitorManager , + entityCollectorCreator ); } diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageEntityCollector.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageEntityCollector.java new file mode 100644 index 00000000..2bebc8d4 --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageEntityCollector.java @@ -0,0 +1,236 @@ +package org.eclipse.store.storage.types; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import org.eclipse.serializer.functional._longProcedure; +import org.eclipse.serializer.persistence.binary.types.ChunksBuffer; +import org.eclipse.serializer.util.logging.Logging; +import org.eclipse.store.storage.exceptions.StorageExceptionConsistency; +import org.eclipse.store.storage.types.StorageEntityCache.Default; +import org.slf4j.Logger; + +public interface StorageEntityCollector extends _longProcedure +{ + + /** + * Responsible to create the StorageEntityCollector used by the storage + * on standard load operations. + */ + public interface Creator + { + public static Creator Default() + { + return new EntityCollectorCreatorByOid(); + } + + public static Creator Unchecked() + { + return new EntityCollectorCreatorByOidUnchecked(); + } + + StorageEntityCollector create(StorageEntityCache.Default entityCache, ChunksBuffer dataCollector); + + public class EntityCollectorCreatorByOid implements Creator + { + @Override + public StorageEntityCollector create(Default entityCache, ChunksBuffer dataCollector) + { + return new EntityCollectorByOid(entityCache, dataCollector); + } + } + + public class EntityCollectorCreatorByOidUnchecked implements Creator + { + @Override + public StorageEntityCollector create(Default entityCache, ChunksBuffer dataCollector) + { + return new EntityCollectorByOidUnchecked(entityCache, dataCollector); + } + } + + } + + /** + * Default StorageEntityCollector implementation that will fail + * with a StorageExceptionConsistency exception if the storage + * does not contain a persisted object with the given id. + */ + class EntityCollectorByOid implements StorageEntityCollector + { + // (01.06.2013 TM)TODO: clean up / consolidate all internal implementations + + /////////////////////////////////////////////////////////////////////////// + // instance fields // + //////////////////// + + private final StorageEntityCache.Default entityCache ; + private final ChunksBuffer dataCollector; + + + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public EntityCollectorByOid( + final StorageEntityCache.Default entityCache , + final ChunksBuffer dataCollector + ) + { + super(); + this.entityCache = entityCache ; + this.dataCollector = dataCollector; + } + + + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public final void accept(final long objectId) + { + final StorageEntity.Default entry; + if((entry = this.entityCache.getEntry(objectId)) == null) + { + /* (14.01.2015 TM)NOTE: this actually is an error, as every oid request comes + * from a referencing entity from inside the same database. So if any load request lookup + * yields null, it is an inconsistency that has to be expressed rather sooner than later. + * + * If some kind of querying request (look if an arbitrary oid yields an entity) is needed, + * it has to be a dedicated kind of request, not this one. + * This one does recursive graph loading (consistency required), not arbitrary querying + * with optional results. + */ + + throw new StorageExceptionConsistency("No entity found for objectId " + objectId); + } + entry.copyCachedData(this.dataCollector); + this.entityCache.checkForCacheClear(entry, System.currentTimeMillis()); + } + + } + + /** + * Special StorageEntityCollector implementation that will NOT fail + * with a StorageExceptionConsistency exception if the storage + * does not contain a persisted object with the given id. + * Instead, the missing objects id will only be logged. + *

+ * Use this StorageEntityCollector with extreme caution as it may + * result in more unrecognized persistence errors and missing runtime objects + * when used wrong! + */ + class EntityCollectorByOidUnchecked implements StorageEntityCollector + { + private final static Logger logger = Logging.getLogger(StorageEntityCollector.class); + + /////////////////////////////////////////////////////////////////////////// + // instance fields // + //////////////////// + + private final StorageEntityCache.Default entityCache ; + private final ChunksBuffer dataCollector; + + + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public EntityCollectorByOidUnchecked( + final StorageEntityCache.Default entityCache , + final ChunksBuffer dataCollector + ) + { + super(); + this.entityCache = entityCache ; + this.dataCollector = dataCollector; + } + + + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public final void accept(final long objectId) + { + final StorageEntity.Default entry; + if((entry = this.entityCache.getEntry(objectId)) == null) + { + logger.warn("No entity found for ObjectID {}, continuing without throwing an exception!", objectId); + return; + } + entry.copyCachedData(this.dataCollector); + this.entityCache.checkForCacheClear(entry, System.currentTimeMillis()); + } + + } + + class EntityCollectorByTid implements StorageEntityCollector + { + /////////////////////////////////////////////////////////////////////////// + // instance fields // + //////////////////// + + private final StorageEntityCache.Default entityCache ; + private final ChunksBuffer dataCollector; + + + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public EntityCollectorByTid( + final StorageEntityCache.Default entityCache , + final ChunksBuffer dataCollector + ) + { + super(); + this.entityCache = entityCache ; + this.dataCollector = dataCollector; + } + + + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public final void accept(final long tid) + { + final StorageEntityType.Default type; + if((type = this.entityCache.getType(tid)) == null) + { + // it can very well be that a channel does not have a certain type at all. That is no error + return; + } + + // all the type's entities are iterated and their data is collected + for(StorageEntity.Default entity = type.head; (entity = entity.typeNext) != null;) + { + entity.copyCachedData(this.dataCollector); + this.entityCache.checkForCacheClear(entity, System.currentTimeMillis()); + } + } + + } + +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageFoundation.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageFoundation.java index 32b196b8..97d67688 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageFoundation.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageFoundation.java @@ -66,7 +66,7 @@ public interface StorageFoundation> extends Insta * Returns the currently set {@link StorageConfiguration} instance. *

* If no instance is set and the implementation deems an instance of this type mandatory for the successful - * executon of {@link #createStorageSystem()}, a suitable instance is created via an internal default + * execution of {@link #createStorageSystem()}, a suitable instance is created via an internal default * creation logic and then set as the current. If the implementation has not sufficient logic and/or data * to create a default instance, a {@link MissingFoundationPartException} is thrown. * @@ -94,7 +94,7 @@ public interface StorageFoundation> extends Insta * Returns the currently set {@link StorageInitialDataFileNumberProvider} instance. *

* If no instance is set and the implementation deems an instance of this type mandatory for the successful - * executon of {@link #createStorageSystem()}, a suitable instance is created via an internal default + * execution of {@link #createStorageSystem()}, a suitable instance is created via an internal default * creation logic and then set as the current. If the implementation has not sufficient logic and/or data * to create a default instance, a {@link MissingFoundationPartException} is thrown. * @@ -526,6 +526,20 @@ public interface StorageFoundation> extends Insta public MonitoringManager getStorageMonitorManager(); + /** + * Returns the currently set {@link StorageEntityCollector.Creator} instance. + *

+ * If no instance is set and the implementation deems an instance of this type mandatory for the successful + * execution of {@link #createStorageSystem()}, a suitable instance is created via an internal default + * creation logic and then set as the current. If the implementation has not sufficient logic and/or data + * to create a default instance, a {@link MissingFoundationPartException} is thrown. + * + * @return the currently set instance, potentially created on-demand if required. + * + * @throws MissingFoundationPartException if a returnable instance is required but cannot be created by default. + */ + public StorageEntityCollector.Creator getStorageEntityCollectorCreator(); + /** * Sets the {@link StorageConfiguration} instance to be used for the assembly. * @@ -838,6 +852,15 @@ public default F setEventLogger(final StorageEventLogger eventLogger) public F setStorageMonitorManager(MonitoringManager storageMonitorManager); + /** + * Sets the {@link StorageEntityCollector.Creator} instance to be used for the assembly. + * + * @param storageEntityCollectorCreator the instance to be used. + * + * @return {@literal this} to allow method chaining. + */ + public F setStorageEntityCollectorCreator(StorageEntityCollector.Creator storageEntityCollectorCreator); + /** * Creates and returns a new {@link StorageSystem} instance by using the current state of all registered * logic part instances and by on-demand creating missing ones via a default logic. @@ -903,6 +926,7 @@ public class Default> private Reference storerRegistryReference ; private StorageStructureValidator storageStructureValidator ; private MonitoringManager storageMonitorManager ; + private StorageEntityCollector.Creator storageEntityCollectorCreator; @@ -1157,6 +1181,11 @@ protected MonitoringManager ensureStorageMonitorManager() return MonitoringManager.PlatformDependent(null); } + protected StorageEntityCollector.Creator ensureStorageEntityCollectorCreator() + { + return StorageEntityCollector.Creator.Default(); + } + @Override public StorageOperationController.Creator getOperationControllerCreator() @@ -1548,6 +1577,16 @@ public MonitoringManager getStorageMonitorManager() return this.storageMonitorManager; } + @Override + public StorageEntityCollector.Creator getStorageEntityCollectorCreator() + { + if(this.storageEntityCollectorCreator == null) + { + this.storageEntityCollectorCreator = this.dispatch(this.ensureStorageEntityCollectorCreator()); + } + return this.storageEntityCollectorCreator; + } + @Override public F setOperationControllerCreator( @@ -1855,10 +1894,17 @@ public F setStorageMonitorManager(final MonitoringManager storageMonitorManager) return this.$(); } + @Override + public final F setStorageEntityCollectorCreator(final StorageEntityCollector.Creator storageEntityCollectorCreator) + { + this.storageEntityCollectorCreator = storageEntityCollectorCreator; + return this.$(); + } + public final boolean isByteOrderMismatch() { /* (11.02.2019 TM)NOTE: On byte order switching: - * Theoreticaly, the storage engine (OGS) could also use the switchByteOrder mechanism implemented for + * Theoretically, the storage engine (OGS) could also use the switchByteOrder mechanism implemented for * communication (OGC). However, there are a lot stumbling blocks in the details that are currently not * worth resolving for a feature that is most probably never required in the foreseeable future. * See StorageEntityCache$Default#putEntity @@ -1907,7 +1953,8 @@ public StorageSystem createStorageSystem() this.getLiveObjectIdChecker() , this.getLiveStorerRegistryReference() , this.getStorageStructureValidator() , - this.getStorageMonitorManager() + this.getStorageMonitorManager() , + this.getStorageEntityCollectorCreator() ); } diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageSystem.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageSystem.java index 5dbe98c0..f6496209 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageSystem.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageSystem.java @@ -189,6 +189,7 @@ public final class Default implements StorageSystem, Unpersistable, StorageKilla private final boolean switchByteOrder ; private final StorageStructureValidator storageStructureValidator ; private final MonitoringManager monitorManager ; + private final StorageEntityCollector.Creator entityCollectorCreator ; // state flags // private final AtomicBoolean isStartingUp = new AtomicBoolean(); @@ -244,7 +245,8 @@ public Default( final ObjectIdsSelector liveObjectIdChecker , final Referencing refStorerRegistry , final StorageStructureValidator storageStructureValidator , - final MonitoringManager monitorManager + final MonitoringManager monitorManager , + final StorageEntityCollector.Creator entityCollectorCreator ) { super(); @@ -291,6 +293,7 @@ public Default( this.switchByteOrder = switchByteOrder ; this.storageStructureValidator = notNull(storageStructureValidator) ; this.monitorManager = notNull(monitorManager) ; + this.entityCollectorCreator = notNull(entityCollectorCreator) ; } @@ -550,7 +553,8 @@ private void createChannels() this.refStorerRegistry , this.switchByteOrder , this.rootTypeIdProvider.provideRootTypeId(), - this.monitorManager + this.monitorManager , + this.entityCollectorCreator ); final ChannelKeeper[] keepers = this.channelKeepers; From ba7ae1ec3b21a89c95d4480b667fc78ff2b86542 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Wed, 28 May 2025 12:42:53 +0200 Subject: [PATCH 08/14] Feature/export adjacency data (#408) * exporting storage graph adjacency data --- .../cdi/types/config/StorageManagerProxy.java | 9 + .../types/EmbeddedStorageManager.java | 10 + .../types/StorageAdjacencyDataExporter.java | 259 ++++++++++++++++++ .../store/storage/types/StorageChannel.java | 21 ++ .../storage/types/StorageConnection.java | 42 +++ .../storage/types/StorageRequestAcceptor.java | 13 + .../types/StorageRequestTaskCreator.java | 23 ++ ...StorageRequestTaskExportAdjacencyData.java | 93 +++++++ .../storage/types/StorageTaskBroker.java | 17 ++ 9 files changed, 487 insertions(+) create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/types/StorageAdjacencyDataExporter.java create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskExportAdjacencyData.java diff --git a/integrations/cdi4/src/main/java/org/eclipse/store/integrations/cdi/types/config/StorageManagerProxy.java b/integrations/cdi4/src/main/java/org/eclipse/store/integrations/cdi/types/config/StorageManagerProxy.java index 7901bfae..60cb2d7e 100644 --- a/integrations/cdi4/src/main/java/org/eclipse/store/integrations/cdi/types/config/StorageManagerProxy.java +++ b/integrations/cdi4/src/main/java/org/eclipse/store/integrations/cdi/types/config/StorageManagerProxy.java @@ -16,6 +16,8 @@ import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.List; import java.util.Optional; import java.util.function.Predicate; @@ -34,6 +36,7 @@ import org.eclipse.store.storage.embedded.types.EmbeddedStorageFoundation; import org.eclipse.store.storage.embedded.types.EmbeddedStorageManager; import org.eclipse.store.storage.types.Database; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; import org.eclipse.store.storage.types.StorageConfiguration; import org.eclipse.store.storage.types.StorageConnection; import org.eclipse.store.storage.types.StorageEntityCacheEvaluator; @@ -258,6 +261,12 @@ public StorageRawFileStatistics createStorageStatistics() return this.getStorageManager().createStorageStatistics(); } + @Override + public List exportAdjacencyData(final Path workingDir) + { + return this.getStorageManager().exportAdjacencyData(workingDir); + } + @Override public void exportChannels(final StorageLiveFileProvider fileProvider, final boolean performGarbageCollection) { diff --git a/storage/embedded/src/main/java/org/eclipse/store/storage/embedded/types/EmbeddedStorageManager.java b/storage/embedded/src/main/java/org/eclipse/store/storage/embedded/types/EmbeddedStorageManager.java index 491d4482..45494d61 100644 --- a/storage/embedded/src/main/java/org/eclipse/store/storage/embedded/types/EmbeddedStorageManager.java +++ b/storage/embedded/src/main/java/org/eclipse/store/storage/embedded/types/EmbeddedStorageManager.java @@ -17,7 +17,9 @@ import static org.eclipse.serializer.util.X.notNull; import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.Arrays; +import java.util.List; import java.util.function.Predicate; import org.eclipse.serializer.afs.types.AFile; @@ -45,6 +47,7 @@ import org.eclipse.store.storage.monitoring.ObjectRegistryMonitor; import org.eclipse.store.storage.monitoring.StorageManagerMonitor; import org.eclipse.store.storage.types.Database; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; import org.eclipse.store.storage.types.StorageConfiguration; import org.eclipse.store.storage.types.StorageConnection; import org.eclipse.store.storage.types.StorageEntityCacheEvaluator; @@ -540,6 +543,13 @@ public void issueTransactionsLogCleanup() this.singletonConnection().issueTransactionsLogCleanup(); } + @Override + public List exportAdjacencyData( + final Path workingDir) + { + return this.singletonConnection().exportAdjacencyData(workingDir); + } + @Override public final StorageRawFileStatistics createStorageStatistics() { diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageAdjacencyDataExporter.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageAdjacencyDataExporter.java new file mode 100644 index 00000000..65ae6fb4 --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageAdjacencyDataExporter.java @@ -0,0 +1,259 @@ +package org.eclipse.store.storage.types; + +import java.io.IOException; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; + +import org.eclipse.serializer.collections.types.XGettingTable; +import org.eclipse.serializer.memory.XMemory; +import org.eclipse.serializer.persistence.binary.types.Binary; +import org.eclipse.serializer.persistence.binary.types.BinaryReferenceTraverser; +import org.eclipse.serializer.persistence.types.Persistence; +import org.eclipse.serializer.persistence.types.PersistenceTypeDefinition; +import org.eclipse.serializer.reference.Swizzling; +import org.eclipse.serializer.typing.KeyValue; +import org.eclipse.serializer.util.logging.Logging; +import org.eclipse.store.storage.exceptions.StorageException; +import org.slf4j.Logger; + +/** + * The StorageAdjacencyDataExporter is intended to export + * all objects references to other objects in the storage. + */ +public interface StorageAdjacencyDataExporter +{ + /** + * Export adjacency data from the provided provided + * storage file. The Objects and references are represented by + * their storage ObjectID. + * + * @param file the storage file to process. + */ + void exportAdjacencyData(StorageLiveDataFile file); + + /** + * Get the current AdjacencyFiles + * + * @return a AdjacencyFiles object containing information on the exported data. + */ + AdjacencyFiles getExportetFiles(); + + + /** + * Defines the result of StorageAdjacencyDataExporter. + */ + public static interface AdjacencyFiles + { + Map get(); + int getChannelIndex(); + + public static class Default implements AdjacencyFiles + { + private final Map processedFiles; + private final int channelIndex; + + public Default(final int channelIndex, final Map processedFiles) + { + this.channelIndex = channelIndex; + this.processedFiles = processedFiles; + } + + @Override + public Map get() + { + return this.processedFiles; + } + + @Override + public int getChannelIndex() + { + return this.channelIndex; + } + } + } + + /** + * Default implementation of the StorageAdjacencyDataExporter interface + */ + public class Default implements StorageAdjacencyDataExporter + { + private final static Logger logger = Logging.getLogger(StorageAdjacencyDataExporter.class); + + private final Path exportDirectory; + private final Hashtable traverser = new Hashtable<>(); + private final XGettingTable typeDefinitions; + + private final Map processedFiles; + private final int channelIndex; + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public Default( + final StorageTypeDictionary typeDictionary, + final Path exportDirectory, + final int channelIndex + ) + { + super(); + this.channelIndex = channelIndex; + this.processedFiles = new HashMap<>(); + this.typeDefinitions = typeDictionary.allTypeDefinitions(); + this.exportDirectory = exportDirectory; + this.createTraverser(); + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public AdjacencyFiles getExportetFiles() + { + return new AdjacencyFiles.Default(this.channelIndex, this.processedFiles); + } + + @Override + public void exportAdjacencyData(final StorageLiveDataFile file) + { + HashMap referenceTable = new HashMap<>(); + + ByteBuffer buffer = XMemory.allocateDirectNative((int)file.size()); + file.readBytes(buffer); + + long startAddress = XMemory.getDirectByteBufferAddress(buffer); + long offset = 0; + + long address = 0; + long size = 0; + long typeID = 0; + long objectID = 0; + + long keyCount = 0; + long refCount = 0; + + while(offset < buffer.limit()) + { + address = startAddress + offset; + size = XMemory.get_long(address); + typeID = XMemory.get_long(address + 8); + objectID = XMemory.get_long(address + 16); + + if(Persistence.IdType.OID.isInRange(objectID)) + { + long[] references = this.getReferenceIDs(address, typeID); + referenceTable.put(objectID, references); + + keyCount++; + refCount+=references.length; + } + + offset += size; + } + + XMemory.deallocateDirectByteBuffer(buffer); + + Path filePath = this.exportDirectory.resolve(file.file().name() + ".bin"); + logger.debug("Exporting reference meta data for file {} to {}", file.identifier(), filePath); + + this.serialize(referenceTable, filePath, keyCount, refCount); + this.processedFiles.put(file.number(), filePath); + } + + + private void serialize( + final HashMap referenceTable, + final Path path, + final long keyCount, + final long refCount) + { + long estimatedbufferSize = ( + (XMemory.byteSize_long() + XMemory.byteSize_int()) * keyCount) + + (refCount * XMemory.byteSizeReference()); + + ByteBuffer buffer = ByteBuffer.allocate((int)estimatedbufferSize); + + for (Map.Entry entry : referenceTable.entrySet()) + { + buffer.putLong(entry.getKey()); + buffer.putInt(entry.getValue().length); + for (long l : entry.getValue()) + { + buffer.putLong(l); + } + } + buffer.flip(); + + + try(FileChannel fc = FileChannel.open(path, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) + { + fc.write(buffer); + } + catch(IOException e) + { + throw new RuntimeException(e); + } + } + + private void createTraverser() + { + for (KeyValue e : this.typeDefinitions) + { + this.traverser.put( + e.key(), + BinaryReferenceTraverser.Static.deriveReferenceTraversers( + e.value().allMembers(), false + )); + } + } + + private long[] getReferenceIDs(final long objectStartAddress, final long typeID) + { + BinaryReferenceTraverser[] tr = this.traverser.get(typeID); + List referencesIDs = new ArrayList<>(); + + if(tr == null) + { + throw new StorageException("No BinaryReferenceTraverser found for typeID " + typeID); + } + + long a = Binary.toEntityContentOffset(objectStartAddress); + + for (BinaryReferenceTraverser binaryReferenceTraverser : tr) + { + a = binaryReferenceTraverser.apply(a, refId -> + { + if (Swizzling.isProperId(refId)) + { + referencesIDs.add(refId); + } + }); + } + + return referencesIDs.stream().mapToLong(l -> l).toArray(); + } + } +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannel.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannel.java index 9d4dfb45..e6dfae6d 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannel.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageChannel.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.function.Predicate; import org.eclipse.serializer.afs.types.AWritableFile; @@ -38,6 +39,7 @@ import org.eclipse.serializer.util.X; import org.eclipse.serializer.util.logging.Logging; import org.eclipse.store.storage.monitoring.StorageChannelHousekeepingMonitor; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; import org.slf4j.Logger; @@ -104,6 +106,8 @@ public StorageIdAnalysis initializeStorage( public void cleanupStore(); + public AdjacencyFiles collectAdjacencyData(Path exportDirectory); + @@ -816,6 +820,23 @@ public final void dispose() this.entityCache.reset(); this.fileManager.dispose(); } + + @Override + public AdjacencyFiles collectAdjacencyData(final Path exportDirectory) { + + logger.debug("Channel {} collecting object references.", this.channelIndex); + + final StorageAdjacencyDataExporter adjacencyDataCollector = new StorageAdjacencyDataExporter.Default( + this.typeDictionary(), + exportDirectory, + this.channelIndex + ); + this.fileManager.iterateStorageFiles(adjacencyDataCollector::exportAdjacencyData); + + logger.debug("Channel {} collecting object references finished.", this.channelIndex); + + return adjacencyDataCollector.getExportetFiles(); + } } @FunctionalInterface diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageConnection.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageConnection.java index 4a361d6b..dbc5536a 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageConnection.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageConnection.java @@ -17,6 +17,8 @@ import static org.eclipse.serializer.util.X.notNull; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.List; import java.util.function.Predicate; import org.eclipse.serializer.afs.types.ADirectory; @@ -24,6 +26,8 @@ import org.eclipse.serializer.collections.types.XGettingEnum; import org.eclipse.serializer.persistence.binary.types.Binary; import org.eclipse.serializer.persistence.types.PersistenceManager; +import org.eclipse.serializer.persistence.types.PersistenceObjectRegistry; +import org.eclipse.serializer.persistence.types.PersistenceRootsView; import org.eclipse.serializer.persistence.types.PersistenceTypeDictionaryExporter; import org.eclipse.serializer.persistence.types.PersistenceTypeDictionaryFileHandler; import org.eclipse.serializer.persistence.types.Persister; @@ -31,6 +35,7 @@ import org.eclipse.serializer.persistence.types.Unpersistable; import org.eclipse.serializer.reference.UsageMarkable; import org.eclipse.store.storage.exceptions.StorageExceptionBackupFullBackupTargetNotEmpty; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; /** @@ -248,6 +253,15 @@ public void issueFullBackup( */ public void issueTransactionsLogCleanup(); + /** + * Export storage graph adjacecy data to the specified directory. + * The directory must exist! + * + * @param workingDir export directory. + * @return StorageAdjacencyDataExporter.AdjacencyFiles holding all exported file paths. + */ + public List exportAdjacencyData(Path workingDir); + /** * Creates a {@link StorageRawFileStatistics} instance, (obviously) containing raw file statistics about * every channel in the storage. @@ -579,6 +593,34 @@ public void issueTransactionsLogCleanup() return; } } + + @Override + public List exportAdjacencyData(final Path workingDir) + { + try + { + final PersistenceObjectRegistry registry = this.persistenceManager().objectRegistry(); + final PersistenceRootsView roots = this.persistenceManager().viewRoots(); + + roots.iterateEntries((s,o) -> { + final long id = registry.lookupObjectId(o); + System.out.println(s + " " + o + ",id: " + id); + }); + + final Object defaultRoot = roots.rootReference().get(); + if(defaultRoot != null) + { + final long rootId = registry.lookupObjectId(defaultRoot); + return this.connectionRequestAcceptor.exportAdjacencyData(rootId, workingDir); + } + } + catch(final InterruptedException e) + { + // thread interrupted, task aborted, return + } + + return null; + } @Override public StorageRawFileStatistics createStorageStatistics() diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestAcceptor.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestAcceptor.java index 21955d57..46b4592c 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestAcceptor.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestAcceptor.java @@ -17,6 +17,8 @@ import static org.eclipse.serializer.util.X.notNull; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.List; import java.util.function.Predicate; import org.eclipse.serializer.afs.types.AFile; @@ -24,6 +26,7 @@ import org.eclipse.serializer.persistence.binary.types.Binary; import org.eclipse.serializer.persistence.types.PersistenceIdSet; import org.eclipse.store.storage.exceptions.StorageExceptionRequest; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; public interface StorageRequestAcceptor @@ -76,6 +79,9 @@ public void issueTransactionsLogCleanup() throws InterruptedException; // exporting // + + public List exportAdjacencyData(long rootID, Path workingDir) + throws InterruptedException; public default StorageEntityTypeExportStatistics exportTypes(final StorageEntityTypeExportFileProvider exportFileProvider) throws InterruptedException @@ -230,6 +236,13 @@ public void issueTransactionsLogCleanup() waitOnTask(this.taskBroker.issueTransactionsLogCleanup()); } + @Override + public List exportAdjacencyData(final long rootId, final Path workingDir) + throws InterruptedException + { + return waitOnTask(this.taskBroker.exportAdjacencyData(workingDir)).result(); + } + @Override public final StorageEntityTypeExportStatistics exportTypes( final StorageEntityTypeExportFileProvider exportFileProvider, diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskCreator.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskCreator.java index 82cc7d85..f5813799 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskCreator.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskCreator.java @@ -17,6 +17,7 @@ import static org.eclipse.serializer.util.X.notNull; import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.function.Predicate; import org.eclipse.serializer.afs.types.AFile; @@ -97,6 +98,13 @@ public StorageRequestTaskTransactionsLogCleanup CreateTransactionsLogCleanupTask StorageOperationController operationController ); + public StorageRequestTaskExportAdjacencyData createExportAdjacencyDataTask + ( + int channelCount, + StorageOperationController operationController, + Path workingDir + ); + public StorageRequestTaskImportDataFiles createImportFromFilesTask( int channelCount , StorageDataFileEvaluator fileEvaluator , @@ -315,6 +323,21 @@ public StorageRequestTaskTransactionsLogCleanup CreateTransactionsLogCleanupTask ); } + @Override + public StorageRequestTaskExportAdjacencyData createExportAdjacencyDataTask( + final int channelCount, + final StorageOperationController operationController, + final Path exportDirectory + ) + { + return new StorageRequestTaskExportAdjacencyData.Default( + this.timestampProvider.currentNanoTimestamp(), + channelCount, + operationController, + exportDirectory + ); + } + @Override public StorageRequestTaskImportDataFiles createImportFromFilesTask( final int channelCount , diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskExportAdjacencyData.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskExportAdjacencyData.java new file mode 100644 index 00000000..07b1def4 --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskExportAdjacencyData.java @@ -0,0 +1,93 @@ +package org.eclipse.store.storage.types; + +import java.nio.file.Files; +import java.nio.file.Path; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.util.Arrays; +import java.util.List; + +import org.eclipse.store.storage.exceptions.StorageException; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; + +/** + * Collect persisted object and reference data of all storage channels. + * See {@link StorageAdjacencyDataExporter}. + */ +public interface StorageRequestTaskExportAdjacencyData extends StorageRequestTask +{ + /** + * Return list of maps with missing object Ids + * and the reference chain to those. + * + * @return a list of maps. + */ + public List result(); + + public final class Default + extends StorageChannelSynchronizingTask.AbstractCompletingTask + implements StorageRequestTaskExportAdjacencyData, StorageChannelTask + { + private final List channelResults; + private final Path exportDirectory; + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public Default(final long timestamp, final int channelCount, final StorageOperationController controller, final Path exportDirectory) + { + super(timestamp, channelCount, controller); + this.channelResults = Arrays.asList(null, null, null, null); + this.exportDirectory = this.ensureDirectory(exportDirectory); + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + protected final Void internalProcessBy(final StorageChannel channel) + { + final AdjacencyFiles result = channel.collectAdjacencyData(this.exportDirectory); + this.channelResults.set(channel.channelIndex(), result); + + return null; + } + + @Override + protected void postCompletionSuccess(final StorageChannel channel, final Void result) throws InterruptedException + { + super.postCompletionSuccess(channel, result); + } + + @Override + public final List result() + { + return this.channelResults; + } + + private Path ensureDirectory(final Path exportDirectory) + { + if(!Files.exists(exportDirectory)) + { + throw new StorageException("Directory not found: " + exportDirectory); + } + + return exportDirectory; + } + } +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageTaskBroker.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageTaskBroker.java index 8c947a5f..0e5de483 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageTaskBroker.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageTaskBroker.java @@ -17,6 +17,7 @@ import static org.eclipse.serializer.util.X.notNull; import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.function.Predicate; import org.eclipse.serializer.afs.types.AFile; @@ -97,6 +98,9 @@ public StorageRequestTaskCacheCheck issueCacheCheck( public StorageRequestTaskTransactionsLogCleanup issueTransactionsLogCleanup() throws InterruptedException; + public StorageRequestTaskExportAdjacencyData exportAdjacencyData(Path workingDir) + throws InterruptedException; + public StorageOperationController operationController(); public final class Default implements StorageTaskBroker @@ -287,6 +291,19 @@ public final synchronized StorageRequestTaskTransactionsLogCleanup issueTransact return task; } + @Override + public final synchronized StorageRequestTaskExportAdjacencyData exportAdjacencyData(final Path exportDirectory) + throws InterruptedException + { + final StorageRequestTaskExportAdjacencyData task = this.taskCreator.createExportAdjacencyDataTask( + this.channelCount, + this.operationController, + exportDirectory + ); + this.enqueueTaskAndNotifyAll(task); + return task; + } + @Override public final synchronized StorageRequestTask enqueueExportChannelsTask( final StorageLiveFileProvider fileProvider , From ff2dc2f9b46f6007636f08b03cccbb2622b0ec4b Mon Sep 17 00:00:00 2001 From: Zdenek Jonas Date: Tue, 24 Jun 2025 11:51:31 +0200 Subject: [PATCH 09/14] maven central migrate (#416) * chore: update Maven deployment configurations for central repository * chore: update Maven snapshot repository URL to central Sonatype * chore: fix syntax error in maven_deploy_snapshot_dev.yml --- .github/workflows/maven_deploy_snapshot.yml | 8 +++---- .../workflows/maven_deploy_snapshot_dev.yml | 8 +++---- pom.xml | 21 +++++++++++++------ 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/workflows/maven_deploy_snapshot.yml b/.github/workflows/maven_deploy_snapshot.yml index 6a88aecf..95584ce1 100644 --- a/.github/workflows/maven_deploy_snapshot.yml +++ b/.github/workflows/maven_deploy_snapshot.yml @@ -27,8 +27,8 @@ jobs: - name: Make a snapshot run: mvn -Pdeploy -Pproduction --no-transfer-progress --batch-mode clean deploy -U env: - MAVEN_USERNAME: ${{ secrets.ORG_OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.ORG_OSSRH_PASSWORD }} + MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} @@ -62,7 +62,7 @@ jobs: mvn -Pdeploy -Pproduction -pl storage/rest/client-app-standalone-assembly clean deploy mvn -Pdeploy -Pproduction -pl storage/rest/service-springboot clean deploy env: - MAVEN_USERNAME: ${{ secrets.ORG_OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.ORG_OSSRH_PASSWORD }} + MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} diff --git a/.github/workflows/maven_deploy_snapshot_dev.yml b/.github/workflows/maven_deploy_snapshot_dev.yml index 56274af8..83a97c8c 100644 --- a/.github/workflows/maven_deploy_snapshot_dev.yml +++ b/.github/workflows/maven_deploy_snapshot_dev.yml @@ -47,8 +47,8 @@ jobs: - name: Make a snapshot run: mvn -Pdeploy -Pproduction --no-transfer-progress --batch-mode clean deploy -U env: - MAVEN_USERNAME: ${{ secrets.ORG_OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.ORG_OSSRH_PASSWORD }} + MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} @@ -103,7 +103,7 @@ jobs: mvn -Pdeploy -Pproduction -pl storage/rest/client-app-standalone-assembly clean deploy mvn -Pdeploy -Pproduction -pl storage/rest/service-springboot clean deploy env: - MAVEN_USERNAME: ${{ secrets.ORG_OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.ORG_OSSRH_PASSWORD }} + MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} diff --git a/pom.xml b/pom.xml index 5f2f4e7f..9dd20d1e 100644 --- a/pom.xml +++ b/pom.xml @@ -69,18 +69,14 @@ ossrh - https://oss.sonatype.org/content/repositories/snapshots + https://central.sonatype.com/repository/maven-snapshots - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - ossrh - https://oss.sonatype.org/content/repositories/snapshots + https://central.sonatype.com/repository/maven-snapshots false @@ -152,6 +148,10 @@ org.apache.maven.plugins maven-enforcer-plugin + + org.sonatype.central + central-publishing-maven-plugin + @@ -377,6 +377,15 @@ ${project.build.directory}/sbom + + org.sonatype.central + central-publishing-maven-plugin + 0.8.0 + true + + central + + From f75b9f5c281671b8ee9356b71d115d8907ee34b0 Mon Sep 17 00:00:00 2001 From: Zdenek Jonas Date: Thu, 26 Jun 2025 09:17:35 +0200 Subject: [PATCH 10/14] chore: update build configurations to use JDK 17 (#417) --- .github/workflows/maven_build.yml | 12 ---- .github/workflows/maven_build_win.yml | 13 +---- .github/workflows/maven_converter.yml | 5 +- .github/workflows/maven_deploy_snapshot.yml | 36 +----------- .../workflows/maven_deploy_snapshot_dev.yml | 57 +------------------ integrations/itest/pom.xml | 1 + integrations/pom.xml | 37 ++---------- integrations/spring-boot3-console/pom.xml | 1 + integrations/spring-boot3/pom.xml | 1 + pom.xml | 3 +- .../client-app-standalone-assembly/pom.xml | 1 + storage/rest/client-app/pom.xml | 1 + storage/rest/pom.xml | 36 +----------- storage/rest/service-springboot/pom.xml | 1 + 14 files changed, 19 insertions(+), 186 deletions(-) diff --git a/.github/workflows/maven_build.yml b/.github/workflows/maven_build.yml index ac7813b4..70a16143 100644 --- a/.github/workflows/maven_build.yml +++ b/.github/workflows/maven_build.yml @@ -14,18 +14,6 @@ jobs: build: runs-on: ubuntu-latest steps: - #Build with java 11 - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - cache: 'maven' - - name: Build with Maven - run: mvn -P examples -B clean package --file pom.xml -U - - #Build with java 17 - uses: actions/checkout@v3 - name: Set up JDK 17 uses: actions/setup-java@v3 diff --git a/.github/workflows/maven_build_win.yml b/.github/workflows/maven_build_win.yml index 2411e0e7..912eb9f4 100644 --- a/.github/workflows/maven_build_win.yml +++ b/.github/workflows/maven_build_win.yml @@ -14,18 +14,6 @@ jobs: build: runs-on: windows-latest steps: - #Build with java 11 - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - cache: 'maven' - - name: Build with Maven - run: mvn -P examples -B clean package --file pom.xml -U - - #Build with java 17 - uses: actions/checkout@v3 - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -35,3 +23,4 @@ jobs: cache: 'maven' - name: Build with Maven run: mvn -P examples -B clean package --file pom.xml -U + diff --git a/.github/workflows/maven_converter.yml b/.github/workflows/maven_converter.yml index 104fa5c8..3c056ae2 100644 --- a/.github/workflows/maven_converter.yml +++ b/.github/workflows/maven_converter.yml @@ -14,12 +14,11 @@ jobs: build: runs-on: ubuntu-latest steps: - #Build with java 11 - uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '17' distribution: 'adopt' - name: Build with Maven run: mvn -B -Pconverter-standalone -pl storage/embedded-tools/storage-converter -am clean package --file pom.xml -U diff --git a/.github/workflows/maven_deploy_snapshot.yml b/.github/workflows/maven_deploy_snapshot.yml index 95584ce1..472ee0ee 100644 --- a/.github/workflows/maven_deploy_snapshot.yml +++ b/.github/workflows/maven_deploy_snapshot.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Java for publishing to Maven Central Snapshot Repository uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' cache: 'maven' server-id: ossrh @@ -32,37 +32,3 @@ jobs: MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} - #java 17 build - publish_java17: - if: github.repository == 'eclipse-store/store' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Java 17 for publishing to Maven Central Repository - uses: actions/setup-java@v3 - with: - java-version: '17' - distribution: 'temurin' - cache: 'maven' - server-id: ossrh - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - - name: Build with java 17 - run: | - mvn -P production -pl integrations/spring-boot3 clean install -am -B - mvn -P production -pl integrations/spring-boot3-console clean install -am -B - mvn -P production -pl storage/rest/client-app clean install -am -B - mvn -P production -pl storage/rest/client-app-standalone-assembly clean install -am -B - mvn -P production -pl storage/rest/service-springboot clean install -am -B - - name: Deploy module build with java 17 - run: | - mvn -Pdeploy -Pproduction -pl integrations/spring-boot3 clean deploy - mvn -Pdeploy -Pproduction -pl integrations/spring-boot3-console clean deploy - mvn -Pdeploy -Pproduction -pl storage/rest/client-app clean deploy - mvn -Pdeploy -Pproduction -pl storage/rest/client-app-standalone-assembly clean deploy - mvn -Pdeploy -Pproduction -pl storage/rest/service-springboot clean deploy - env: - MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} - MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} diff --git a/.github/workflows/maven_deploy_snapshot_dev.yml b/.github/workflows/maven_deploy_snapshot_dev.yml index 83a97c8c..9bb68b13 100644 --- a/.github/workflows/maven_deploy_snapshot_dev.yml +++ b/.github/workflows/maven_deploy_snapshot_dev.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Java for publishing to Maven Central Snapshot Repository uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' cache: 'maven' server-id: ossrh @@ -52,58 +52,3 @@ jobs: MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} - - #java 17 build - publish_java17: - if: github.repository == 'eclipse-store/store' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Java for publishing to Maven Central Snapshot Repository - uses: actions/setup-java@v4 - with: - java-version: '17' - distribution: 'temurin' - cache: 'maven' - server-id: ossrh - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - - name: Prepare suffix - run: | - suffix=$(echo -n "${GITHUB_REF#refs/heads/}" | tr '/' '_' | cut -c1-10)-$(echo -n "${GITHUB_REF#refs/heads/}" | md5sum | cut -c1-10) - echo "Suffix: $suffix" - echo "SUFFIX=$suffix" >> $GITHUB_ENV - - name: Update project version java 17 - run: | - currentVersion=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) - currentVersionWithoutSnapshot=${currentVersion%-SNAPSHOT} - newVersion="${currentVersionWithoutSnapshot}-$SUFFIX-SNAPSHOT" - mvn versions:set -DnewVersion=$newVersion --batch-mode - BRANCH_NAME=${{ github.ref }} - REPO_OWNER="eclipse-serializer" - REPO_NAME="serializer" - RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/$REPO_OWNER/$REPO_NAME/branches/$BRANCH_NAME) - if [ $RESPONSE -eq 200 ]; then - mvn versions:set-property -Dproperty=eclipse.serializer.version -DnewVersion=$newVersion - else - echo "Branch does not exist in serializer repository, skipping serializer version change" - fi - - name: Make a snapshot java 17 - run: | - mvn -P production -pl integrations/spring-boot3 clean install -am -B - mvn -P production -pl integrations/spring-boot3-console clean install -am -B - mvn -P production -pl storage/rest/client-app clean install -am -B - mvn -P production -pl storage/rest/client-app-standalone-assembly clean install -am -B - mvn -P production -pl storage/rest/service-springboot clean install -am -B - - name: Deploy module build with java 17 - run: | - mvn -Pdeploy -Pproduction -pl integrations/spring-boot3 clean deploy - mvn -Pdeploy -Pproduction -pl integrations/spring-boot3-console clean deploy - mvn -Pdeploy -Pproduction -pl storage/rest/client-app clean deploy - mvn -Pdeploy -Pproduction -pl storage/rest/client-app-standalone-assembly clean deploy - mvn -Pdeploy -Pproduction -pl storage/rest/service-springboot clean deploy - env: - MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} - MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} diff --git a/integrations/itest/pom.xml b/integrations/itest/pom.xml index 17fd46b4..7e705d56 100644 --- a/integrations/itest/pom.xml +++ b/integrations/itest/pom.xml @@ -17,6 +17,7 @@ 3.2.2 + 17 diff --git a/integrations/pom.xml b/integrations/pom.xml index 3d0c9d29..7a10ca96 100644 --- a/integrations/pom.xml +++ b/integrations/pom.xml @@ -18,39 +18,10 @@ cdi4 + spring-boot3 + spring-boot3-console + + itest - - - from_java_17 - - [17,) - - - spring-boot3 - spring-boot3-console - - itest - - - 17 - 17 - 17 - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - 17 - - - - - - - - diff --git a/integrations/spring-boot3-console/pom.xml b/integrations/spring-boot3-console/pom.xml index 4cb3fe32..598eace9 100644 --- a/integrations/spring-boot3-console/pom.xml +++ b/integrations/spring-boot3-console/pom.xml @@ -17,6 +17,7 @@ 3.4.5 + 17 diff --git a/integrations/spring-boot3/pom.xml b/integrations/spring-boot3/pom.xml index fa15453d..dc9acd53 100644 --- a/integrations/spring-boot3/pom.xml +++ b/integrations/spring-boot3/pom.xml @@ -17,6 +17,7 @@ 3.4.5 + 17 diff --git a/pom.xml b/pom.xml index 9dd20d1e..9b0883d9 100644 --- a/pom.xml +++ b/pom.xml @@ -36,8 +36,7 @@ UTF-8 - 11 - 11 + 11 javac 3.8.1 11 diff --git a/storage/rest/client-app-standalone-assembly/pom.xml b/storage/rest/client-app-standalone-assembly/pom.xml index ec465e6a..420d74a1 100644 --- a/storage/rest/client-app-standalone-assembly/pom.xml +++ b/storage/rest/client-app-standalone-assembly/pom.xml @@ -21,6 +21,7 @@ 24.2.6 3.2.0 + 17 diff --git a/storage/rest/client-app/pom.xml b/storage/rest/client-app/pom.xml index 70280efd..ad9790dc 100644 --- a/storage/rest/client-app/pom.xml +++ b/storage/rest/client-app/pom.xml @@ -22,6 +22,7 @@ 24.2.6 3.2.0 + 17 diff --git a/storage/rest/pom.xml b/storage/rest/pom.xml index c2fafbb9..0ad10a33 100644 --- a/storage/rest/pom.xml +++ b/storage/rest/pom.xml @@ -26,39 +26,9 @@ client-jersey service service-sparkjava + service-springboot + client-app + client-app-standalone-assembly - - - from_java_17 - - [17,) - - - 17 - 17 - 17 - - - service-springboot - client-app - client-app-standalone-assembly - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - 17 - - - - - - - - - diff --git a/storage/rest/service-springboot/pom.xml b/storage/rest/service-springboot/pom.xml index 00ae0150..10dd5cee 100644 --- a/storage/rest/service-springboot/pom.xml +++ b/storage/rest/service-springboot/pom.xml @@ -21,6 +21,7 @@ 3.2.2 + 17 From 1604cdd3a79b84d972b20a7478c7e8109dc82db5 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Thu, 26 Jun 2025 14:38:08 +0200 Subject: [PATCH 11/14] Feature: Storage Structural Reference Analysis (#413) Feature: Storage Structural Reference Analysis --- .../storage/src/main/java/module-info.java | 3 +- .../analysis/AdjacencyDataConverter.java | 437 +++++++++++++++ .../store/storage/analysis/AdjacencyMap.java | 187 +++++++ .../store/storage/analysis/AdjacencySet.java | 222 ++++++++ .../analysis/ComparableAdjacencyMap.java | 144 +++++ .../storage/analysis/MissingObjects.java | 64 +++ .../analysis/MissingObjectsSearch.java | 527 ++++++++++++++++++ .../store/storage/analysis/ObjectParents.java | 69 +++ .../storage/analysis/ReverseObjectSearch.java | 276 +++++++++ .../types/StorageAdjacencyDataExporter.java | 15 +- 10 files changed, 1939 insertions(+), 5 deletions(-) create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyDataConverter.java create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyMap.java create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencySet.java create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/analysis/ComparableAdjacencyMap.java create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/analysis/MissingObjects.java create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/analysis/MissingObjectsSearch.java create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/analysis/ObjectParents.java create mode 100644 storage/storage/src/main/java/org/eclipse/store/storage/analysis/ReverseObjectSearch.java diff --git a/storage/storage/src/main/java/module-info.java b/storage/storage/src/main/java/module-info.java index 0c33b8de..e446edff 100644 --- a/storage/storage/src/main/java/module-info.java +++ b/storage/storage/src/main/java/module-info.java @@ -17,7 +17,8 @@ exports org.eclipse.store.storage.types; exports org.eclipse.store.storage.exceptions; exports org.eclipse.store.storage.monitoring; - + exports org.eclipse.store.storage.analysis; + requires transitive org.eclipse.store.afs.nio; requires transitive org.eclipse.serializer.persistence.binary; requires transitive org.eclipse.serializer.base; diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyDataConverter.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyDataConverter.java new file mode 100644 index 00000000..4c5f4f0a --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyDataConverter.java @@ -0,0 +1,437 @@ +package org.eclipse.store.storage.analysis; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import org.eclipse.serializer.util.logging.Logging; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; +import org.slf4j.Logger; + + + +/** + * Converts the data exported from {@link StorageAdjacencyDataExporter} + * to different formats used for further processing. + * These are a set of all references in an input file and + * a map of all and there reverencing parents objects. + */ +public interface AdjacencyDataConverter +{ + /** + * Start the data conversion. + * + * @return ConvertedAdjacencyFiles object providing path's to the generated files. + */ + public ConvertedAdjacencyFiles convert(); + + /** + * Holds the path's to adjacency data files produced by the {@link AdjacencyDataConverter}. + */ + public interface ConvertedAdjacencyFiles + { + public List getReferenceSets(); + + public List getReverseReferenceMaps(); + } + + public interface Configuration + { + + } + + public static AdjacencyDataConverter New(final List adjacencyFiles) + { + return new AdjacencyDataConverter.Default(adjacencyFiles); + } + + public static class Default implements AdjacencyDataConverter + { + public static class DefaultConvertedAdjacencyFiles implements ConvertedAdjacencyFiles + { + private final List referenceSets; + private final List reverseReferenceMaps; + + public DefaultConvertedAdjacencyFiles(final List referenceSets, final List referenceMaps) + { + this.referenceSets = referenceSets; + this.reverseReferenceMaps = referenceMaps; + } + + @Override + public final List getReferenceSets() + { + return this.referenceSets; + } + + @Override + public final List getReverseReferenceMaps() + { + return this.reverseReferenceMaps; + } + } + + public static class DefaultConfiguration implements Configuration + { + private final static int THREAD_MINIMUM = 1; + private final static int THREAD_MAXIMUM = 12; + + private final int mapLoaders; + private final int setCreators; + private final int reverseMapCreators; + private final int threadsTotal; + + /** + * Create a configuration object using default values. + */ + public DefaultConfiguration() + { + super(); + this.mapLoaders = 3; + this.setCreators = 3; + this.reverseMapCreators = 3; + + this.threadsTotal = this.mapLoaders + this.setCreators + this.reverseMapCreators; + } + + /** + * Create a new Configuration for the AdjacencyDataConverter. + * This config requires a minimum of one thread and a maximum of 12 for each configuration value. + * + * @param mapLoaders The number of threads used to load adjacency maps during initialisation. + * @param setCreators The number of threads used to create adjacency sets during initialisation. + * @param reverseMapCreators The number of threads used to create reverse adjacency maps during initialisation. + */ + public DefaultConfiguration( + final int mapLoaders, + final int setCreators, + final int reverseMapCreators) + { + super(); + + this.mapLoaders = this.verifyThreadCount(mapLoaders); + this.setCreators = this.verifyThreadCount(setCreators); + this.reverseMapCreators = this.verifyThreadCount(reverseMapCreators); + + this.threadsTotal = this.mapLoaders + this.setCreators + this.reverseMapCreators; + } + + public final int getMapLoaders() + { + return this.mapLoaders; + } + + public final int getSetCreators() + { + return this.setCreators; + } + + public final int getThreadsTotal() + { + return this.threadsTotal; + } + + public final int getReverseMapCreators() + { + return this.reverseMapCreators; + } + + private int verifyThreadCount(final int numThreads) + { + if(numThreads < THREAD_MINIMUM) + { + logger.info("The requested thread count {} is below the required minimum, using minimum default value of {} !", numThreads, THREAD_MINIMUM); + return THREAD_MINIMUM; + } + + if(numThreads > THREAD_MAXIMUM) + { + logger.info("The requested thread count {} is greater then the maximum, using maximum default value of {} !", numThreads, THREAD_MAXIMUM); + return THREAD_MAXIMUM; + } + + return numThreads; + } + } + + private final static Logger logger = Logging.getLogger(MissingObjectsSearch.class); + + private final DefaultConfiguration configuration; + private final ThreadFactory threadFactory; + private final List adjacencyFiles; + private final int fileCount; + private final AtomicInteger setsCreated = new AtomicInteger(); + private final AtomicInteger reverseMapsCreated = new AtomicInteger(); + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public Default(final List adjacencyFiles) + { + this(adjacencyFiles, new DefaultConfiguration()); + } + + public Default(final List adjacencyFiles, final DefaultConfiguration configuration) + { + this.adjacencyFiles = adjacencyFiles; + this.configuration = configuration; + + this.threadFactory = new ThreadFactory() + { + final AtomicInteger counter = new AtomicInteger(); + + @Override + public Thread newThread(final Runnable r) + { + return new Thread(r, "Eclipse-Store-AdjacencyDataConverter-WorkerThread-" + this.counter.getAndIncrement()); + }}; + + int tempFileCount = 0; + for(final AdjacencyFiles channel : this.adjacencyFiles) + { + tempFileCount += channel.get().size(); + } + this.fileCount = tempFileCount; + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + public static Path derivePath(final Path other, final String newExtension) + { + final String fn = other.toString(); + final int index = fn.lastIndexOf('.'); + final String newName = fn.substring(0, index) + newExtension; + return Paths.get(newName); + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public ConvertedAdjacencyFiles convert() + { + final ExecutorService executor = Executors.newFixedThreadPool(this.configuration.getThreadsTotal() , this.threadFactory); + + final LinkedBlockingQueue adjacencyMapsPathsQueue = new LinkedBlockingQueue<>(); + final LinkedBlockingQueue adjacencyMapsQueue1 = new LinkedBlockingQueue<>(this.configuration.getSetCreators()); + final LinkedBlockingQueue adjacencyMapsQueue2 = new LinkedBlockingQueue<>(this.configuration.getReverseMapCreators()); + + for(final AdjacencyFiles channel : this.adjacencyFiles) + { + for(final Entry entry : channel.get().entrySet()) + { + adjacencyMapsPathsQueue.add(entry.getValue()); + } + } + + final List> futures = new ArrayList<>(); + + for(int i = 0; i < this.configuration.getMapLoaders() ; i++) + { + futures.add( + CompletableFuture + .runAsync(() -> Default.adjacencyMapLoader(adjacencyMapsPathsQueue, List.of(adjacencyMapsQueue1, adjacencyMapsQueue2)), executor) + .exceptionally((t) -> shutdownExceptional(executor, t)) + ); + } + + final List referenceMaps = Collections.synchronizedList(new ArrayList<>(this.fileCount)); + for(int i = 0; i < this.configuration.getReverseMapCreators(); i++) + { + futures.add( + CompletableFuture + .runAsync(() -> referenceMaps.addAll(this.reverseReferenceMapCreator(adjacencyMapsQueue1)), executor) + .exceptionally((t) -> shutdownExceptional(executor, t)) + ); + } + + final List referenceSets = Collections.synchronizedList(new ArrayList<>(this.fileCount)); + for(int i = 0; i < this.configuration.getSetCreators(); i++) + { + futures.add( + CompletableFuture + .runAsync(() -> referenceSets.addAll(this.referenceSetCreator(adjacencyMapsQueue2)), executor) + .exceptionally((t) -> shutdownExceptional(executor, t)) + ); + } + + + final CompletableFuture createSetsStage = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + + try + { + createSetsStage.get(); + } + catch(InterruptedException | ExecutionException e) + { + logger.error("prepareReferenceSets failed: ", e); + } + finally + { + shutdown(executor); + } + + return new DefaultConvertedAdjacencyFiles(referenceSets, referenceMaps); + } + + private static Void shutdownExceptional(final ExecutorService ex, final Throwable t) + { + if(ex != null && !ex.isTerminated()) + { + ex.shutdownNow(); + logger.error("Shutting down exceptionally: ", t); + throw new RuntimeException("abnormal termination", t); + } + + return null; + } + + private static Void shutdown(final ExecutorService ex) + { + if(ex != null && !ex.isTerminated()) + { + ex.shutdownNow(); + } + return null; + } + + private List referenceSetCreator( + final LinkedBlockingQueue adjacencyMapsQueue) + { + final List createdFiles = new ArrayList<>(this.fileCount); + + while(this.setsCreated.getAndIncrement() < this.fileCount) + { + try + { + final AdjacencyMap map = adjacencyMapsQueue.take(); + final AdjacencySet rf = new AdjacencySet(map); + + rf.store(); + rf.unload(); + + createdFiles.add(rf.getPath()); + logger.debug("created reference set {}", rf.getPath()); + } + catch(final InterruptedException e) + { + //stop processing + logger.debug("Reference set creator task stopped after interruption"); + return null; + } + } + + logger.debug("Reference set creator task finished successfully."); + return createdFiles; + } + + private List reverseReferenceMapCreator(final LinkedBlockingQueue adjacencyMapsQueue) + { + final List createdFiles = new ArrayList<>(this.fileCount); + + while(this.reverseMapsCreated.getAndIncrement() < this.fileCount) + { + try + { + final AdjacencyMap adjacencyMap = adjacencyMapsQueue.take(); + + final TreeMap map = adjacencyMap.getMap(); + + final Path path = derivePath(adjacencyMap.getPath(), ".brf"); + + + final HashMap> backRefs = new HashMap<>(); + + map.forEach((k,v) -> + { + for(final long r : v) + { + backRefs.computeIfAbsent(r, x -> new HashSet<>()).add(k); + } + }); + + final AtomicLong backRefsCount = new AtomicLong(); + backRefs.values().forEach(v -> backRefsCount.addAndGet(v.size())); + + AdjacencyMap.serialize(backRefs, path, backRefs.size(), backRefsCount.get()); + + createdFiles.add(path); + + logger.debug("created reverse reference map {}", path); + } + catch(final InterruptedException e) + { + //stop processing + logger.debug("Reverse reference map creator task stopped after interruption."); + return null; + } + } + + logger.debug("Reverse reference map creator task finished successfully."); + return createdFiles; + } + + protected static void adjacencyMapLoader( + final LinkedBlockingQueue adjacencyMapsPathsQueue, + final List> outputQueues) + { + while(!adjacencyMapsPathsQueue.isEmpty()) + { + try + { + final Path path = adjacencyMapsPathsQueue.take(); + final TreeMap refMap = AdjacencyMap.deserializeReferenceMap(path); + + for(final LinkedBlockingQueue queue : outputQueues) + { + queue.put(new AdjacencyMap(refMap, path)); + } + + logger.debug("loaded reference map {}", path); + } + catch(final InterruptedException e) + { + //stop processing + logger.debug("Adjacency map loader task stopped after interruption"); + return; + } + } + + logger.debug("Adjacency map loader task finished successfully."); + } + } +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyMap.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyMap.java new file mode 100644 index 00000000..f7f132ea --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyMap.java @@ -0,0 +1,187 @@ +package org.eclipse.store.storage.analysis; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; + +class AdjacencyMap +{ + protected TreeMap map; + final protected Path path; + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public AdjacencyMap(final TreeMap refMap, final Path path) + { + this.map = refMap; + this.path = path; + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + public TreeMap getMap() + { + return this.map; + } + + public Path getPath() + { + return this.path; + } + + /** + * Helper method to deserialize a TreeMap<Long, long[]>> from a file. + * + * @param path the input file + * @return the loaded TreeMap + */ + public static TreeMap deserializeReferenceMap(final Path path) + { + final TreeMap result = new TreeMap<>(); + + try(FileChannel fc = FileChannel.open(path, StandardOpenOption.READ)) + { + final ByteBuffer buffer = ByteBuffer.allocate((int) fc.size()); + fc.read(buffer); + buffer.flip(); + + while(buffer.position() < buffer.limit()) + { + final long k = buffer.getLong(); + final int len = buffer.getInt(); + final long[] v = new long[len]; + for (int i = 0; i < len; i++) + { + v[i] = buffer.getLong(); + } + result.put(k, v); + } + + } + catch(final IOException e) + { + throw new RuntimeException(e); + } + + return result; + } + + /** + * Helper method to serialize a Map<Long, long[]>> to a file. + * + * @param map the map to serialize + * @param path the target file path + */ + public static void serialize(final Map map, final Path path) + { + final long estimatedBufferSize = 1024*1024*4; + + final List buffers = new LinkedList<>(); + + ByteBuffer buffer = ByteBuffer.allocate((int)estimatedBufferSize); + buffers.add(buffer); + + for (final Map.Entry entry : map.entrySet()) + { + final long[] value = entry.getValue(); + final long requiredSize = Long.BYTES + Integer.BYTES + (value.length * Long.BYTES); + + if(buffer.remaining() < requiredSize) + { + buffer.flip(); + buffer = ByteBuffer.allocate((int)requiredSize); + buffers.add(buffer); + } + + buffer.putLong(entry.getKey()); + buffer.putInt(value.length); + for (final long l : value) + { + buffer.putLong(l); + } + } + buffer.flip(); + + try(FileChannel fc = FileChannel.open(path, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) + { + for(final ByteBuffer byteBuffer : buffers) + { + if(fc.write(byteBuffer) < byteBuffer.limit() ) + { + throw new RuntimeException("Failed writing to " + path + ", not all data written!"); + } + } + } + catch(final IOException e) + { + throw new RuntimeException(e); + } + + } + + /** + * Helper method to serialize a Map<Long, Set<Long>> to a file. + * + * @param map the hashmap to serialize + * @param path the target file path + * @param keyCount the number of all keys + * @param valueCount the number of all elements in all value sets + */ + public static void serialize(final Map> map, final Path path, final long keyCount, final long valueCount) + { + final long estimateBufferSize = ((Long.BYTES + Integer.BYTES) * keyCount) + + (valueCount * Long.BYTES); + + final ByteBuffer buffer = ByteBuffer.allocate((int)estimateBufferSize); + + for (final Entry> entry : map.entrySet()) + { + buffer.putLong(entry.getKey()); + buffer.putInt(entry.getValue().size()); + for (final long l : entry.getValue()) + { + buffer.putLong(l); + } + } + buffer.flip(); + + + try(FileChannel fc = FileChannel.open(path, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) + { + if(fc.write(buffer) < buffer.limit() ) + { + throw new RuntimeException("Failed writing to " + path + ", not all data written!"); + } + } + catch(final IOException e) + { + throw new RuntimeException(e); + } + } +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencySet.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencySet.java new file mode 100644 index 00000000..d0857ae1 --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencySet.java @@ -0,0 +1,222 @@ +package org.eclipse.store.storage.analysis; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedByInterruptException; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.eclipse.serializer.persistence.types.Persistence; + +final class AdjacencySet +{ + private ByteBuffer buffer; + private Set references; + private Path path; + private FileChannel fc; + private boolean isEmpty; + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + AdjacencySet(final AdjacencyMap map) + { + super(); + this.init(map); + } + + AdjacencySet(final Path path) + { + super(); + this.path = path; + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + public Path getPath() + { + return this.path; + } + + public Set getReferences() + { + return this.references; + } + + private void init(final AdjacencyMap referenceMap) + { + final Map map = referenceMap.getMap(); + + this.path = AdjacencyDataConverter.Default.derivePath(referenceMap.getPath(), ".ref"); + + this.references = new HashSet<>(); + + map.forEach((k,v) -> { + for(final long r : v) + { + if(Persistence.IdType.OID.isInRange(r)) + { + //referenced id not in this map + if(!map.containsKey(r)) + { + this.getReferences().add(r); + } + } + } + }); + } + + public void load(final boolean truncate) + { + try + { + this.fc = FileChannel.open(this.getPath(), + StandardOpenOption.READ, + StandardOpenOption.WRITE); + + this.buffer = ByteBuffer.allocate((int) this.fc.size()); + this.fc.read(this.buffer); + if(truncate) + { + this.fc.truncate(0); + } + this.buffer.flip(); + + this.references = new HashSet<>(this.buffer.capacity() / Long.BYTES); + + while(this.buffer.position() < this.buffer.limit()) + { + this.getReferences().add(this.buffer.getLong()); + } + this.buffer.clear(); + } + catch(final ClosedByInterruptException e) + { + //suppress ClosedByInterruptException + } + catch(final IOException e) + { + try + { + this.fc.close(); + } + catch(final IOException closeException) + { + //suppress failed file channel close exception + } + throw new RuntimeException(e); + } + } + + public void reduce(final AdjacencyMap map) + { + this.getReferences().removeAll(map.getMap().keySet()); + + if(this.getReferences().isEmpty()) + { + this.isEmpty = true; + } + } + + public void store() + { + if(this.getReferences() == null) + { + return; + } + + if(this.buffer == null) + { + final int size = this.getReferences().size() * Long.BYTES; + this.buffer = ByteBuffer.allocate(size); + } + + for(final long r : this.getReferences()) + { + this.buffer.putLong(r); + } + this.buffer.flip(); + + try + { + if(this.fc == null) + { + this.fc = FileChannel.open(this.getPath(), + StandardOpenOption.CREATE, + StandardOpenOption.READ, + StandardOpenOption.WRITE); + } + + if(this.fc.write(this.buffer) < this.buffer.limit()) + { + throw new RuntimeException("Failed writing to " + this.path + ", not all data written!"); + } + + } + catch(final IOException e) + { + try + { + this.fc.close(); + } + catch(final IOException closeException) + { + //suppress failed file channel close exception + } + + throw new RuntimeException(e); + } + } + + public void unload() + { + this.references = null; + this.buffer = null; + } + + public boolean isEmpty() + { + return this.isEmpty; + } + + public void release() + { + this.unload(); + if(this.fc != null) + { + if(this.fc.isOpen()) + { + try + { + this.fc.close(); + } + catch(final IOException e) + { + //suppress failed file channel close exception + } + } + } + } + +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ComparableAdjacencyMap.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ComparableAdjacencyMap.java new file mode 100644 index 00000000..f1c61676 --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ComparableAdjacencyMap.java @@ -0,0 +1,144 @@ +package org.eclipse.store.storage.analysis; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.nio.file.Path; + +class ComparableAdjacencyMap extends AdjacencyMap implements Comparable +{ + private long size = -1; + private LongRange objectIdRange; + private boolean updated; + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public ComparableAdjacencyMap(final Path path) + { + super(null, path); + this.load(); + this.unload(); + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + public boolean updated() + { + return this.updated; + } + + public boolean inRange(final long value) + { + if(this.objectIdRange != null) + return this.objectIdRange.inRange(value); + return false; + } + + public boolean intersect(final ComparableAdjacencyMap other) + { + return this.objectIdRange.intersect(other.objectIdRange); + } + + public void update() + { + this.size = this.map.size(); + if(!this.isEmpty()) + { + this.objectIdRange = new ComparableAdjacencyMap.LongRange(this.map.firstKey(), this.map.lastKey()); + } + this.updated = true; + } + + public boolean isEmpty() + { + return this.size < 1; + } + + @Override + public String toString() + { + return "ComparableAdjacencyMap [path=" + this.path.getFileName() + ", size=" + this.size + ", objectIdRange=" + this.objectIdRange + "]"; + } + + public void load() + { + this.map = AdjacencyMap.deserializeReferenceMap(this.path); + this.size = this.map.size(); + this.update(); + } + + public void unload() + { + this.map = null; + } + + public void serialize() + { + AdjacencyMap.serialize(this.map, this.path); + this.updated = false; + } + + @Override + public int compareTo(final ComparableAdjacencyMap other) + { + if(this.objectIdRange != null && other.objectIdRange != null) + return this.objectIdRange.compareTo(other.objectIdRange); + return 0; + } + + + private static class LongRange implements Comparable + { + private final long min; + private final long max; + + public LongRange(final long min, final long max) + { + super(); + + if(min > max) throw new IllegalArgumentException("Min value of range is greater then max value!"); + + this.min = min; + this.max = max; + } + + @Override + public String toString() + { + return "LongRange [min=" + this.min + ", max=" + this.max + "]"; + } + + public boolean intersect(final LongRange other) + { + return ((this.min <= other.max) && (this.max >= other.min)); + } + + @Override + public int compareTo(final LongRange other) + { + if(this.min < other.min) return -1; + if(this.min > other.min) return 1; + return Long.compare(this.max, other.max); + } + + public boolean inRange(final long value) + { + return (value >= this.min) && (value <= this.max); + } + } +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/MissingObjects.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/MissingObjects.java new file mode 100644 index 00000000..6941ee6c --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/MissingObjects.java @@ -0,0 +1,64 @@ +package org.eclipse.store.storage.analysis; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.util.Set; + +/** + * Interface defining the result of searching for missing objects. + */ +public interface MissingObjects +{ + /** + * Get all missing IDs. + * + * @return a Set containing all Ids of missing objects. + */ + Set getMissingObjectIDs(); + + /** + * Provides the result of the missing objects search. + */ + class Default implements MissingObjects + { + private final Set missingObjectsIds; + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public Default(final Set missingObjectsIds) + { + super(); + this.missingObjectsIds = missingObjectsIds; + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public Set getMissingObjectIDs() + { + return this.missingObjectsIds; + } + + @Override + public String toString() + { + return "Default [missingObjectsIds=" + this.missingObjectsIds + "]"; + } + } +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/MissingObjectsSearch.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/MissingObjectsSearch.java new file mode 100644 index 00000000..9217d70f --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/MissingObjectsSearch.java @@ -0,0 +1,527 @@ +package org.eclipse.store.storage.analysis; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import org.eclipse.serializer.util.logging.Logging; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; +import org.slf4j.Logger; + +/** + * Searched the collected object reference data form the {@link org.eclipse.store.storage.types.StorageAdjacencyDataExporter} + * for missing objects. + */ +public interface MissingObjectsSearch { + /** + * Search the reference data for missing objects. + * + * @return a set of the missing objets objectIDs. + */ + public MissingObjects searchMissingEntities(); + + public interface Configuration {} + + /** + * Create a new MissingObjectsSearch.Default instance. + * + * @param adjacencyFiles the input adjacency files. + * @param referenceSetsPaths list of path's to the reference set files. + * @param configuration configuration, can be null. + * @return a new MissingObjectsSearch.Default instance. + */ + public static MissingObjectsSearch New( + final List adjacencyFiles, + final List referenceSetsPaths, + final MissingObjectsSearch.Default.DefaultConfiguration configuration) + { + if(configuration != null) + return new MissingObjectsSearch.Default(adjacencyFiles, referenceSetsPaths, configuration); + + return new MissingObjectsSearch.Default(adjacencyFiles, referenceSetsPaths); + } + + public final class Default implements MissingObjectsSearch + { + public static final class DefaultConfiguration implements MissingObjectsSearch.Configuration + { + private final static int THREAD_MINIMUM = 1; + private final static int THREAD_MAXIMUM = 12; + private static final int QUEUE_SIZE = 1; + + private final int reduceStage_MapLoaders; + private final int reduceStage_Initializers; + private final int reduceStage_SetReducers; + private final int reduceStage_TotalThreads; + + /** + * Create a configuration object using default values. + */ + public DefaultConfiguration() + { + super(); + + this.reduceStage_MapLoaders = 1; + this.reduceStage_Initializers = 2; + this.reduceStage_SetReducers = 4; + + this.reduceStage_TotalThreads = this.reduceStage_MapLoaders + this.reduceStage_Initializers + this.reduceStage_SetReducers + 1; + } + + /** + * Create a new Configuration for the MissingObjectsSearch. + * This config requires a minimum of one thread and a maximum of 12 for each configuration value. + * + * @param reduceStage_MapLoaders The number of threads used to load adjacency maps during search stage. + * @param reduceStage_Initializers The number of threads used to load adjacency sets during search stage. + * @param reduceStage_SetReducers The number of threads used to process adjacency sets during search stage. + */ + public DefaultConfiguration( + final int reduceStage_MapLoaders, + final int reduceStage_Initializers, + final int reduceStage_SetReducers) + { + super(); + + this.reduceStage_MapLoaders = this.verifyThreadCount(reduceStage_MapLoaders); + this.reduceStage_Initializers = this.verifyThreadCount(reduceStage_Initializers); + this.reduceStage_SetReducers = this.verifyThreadCount(reduceStage_SetReducers); + + this.reduceStage_TotalThreads = this.reduceStage_MapLoaders + this.reduceStage_Initializers + this.reduceStage_SetReducers + 1; + } + + public final int getReduceStage_MapLoaders() + { + return this.reduceStage_MapLoaders; + } + + public final int getReduceStage_Initializers() + { + return this.reduceStage_Initializers; + } + + public final int getReduceStage_SetReducers() + { + return this.reduceStage_SetReducers; + } + + public final int getReduceStage_TotalThreads() + { + return this.reduceStage_TotalThreads; + } + + public final int getQUEUE_SIZE() + { + return DefaultConfiguration.QUEUE_SIZE; + } + + private int verifyThreadCount(final int numThreads) + { + if(numThreads < THREAD_MINIMUM) + { + logger.info("The requested thread count {} is below the required minimum, using minimum default value of {} !", numThreads, THREAD_MINIMUM); + return THREAD_MINIMUM; + } + + if(numThreads > THREAD_MAXIMUM) + { + logger.info("The requested thread count {} is greater then the maximum, using maximum default value of {} !", numThreads, THREAD_MAXIMUM); + return THREAD_MAXIMUM; + } + + return numThreads; + } + + } + + /////////////////////////////////////////////////////////////////////////// + // Fields // + /////////// + + private final static Logger logger = Logging.getLogger(MissingObjectsSearch.class); + + private final DefaultConfiguration configuration; + + private final List adjacencyFiles; + private final List referenceSetsPaths; + private long fileCount; + + private final ThreadFactory threadFactory; + + private final AtomicInteger mapsTaken = new AtomicInteger(); + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + /** + * Create a new MissingObjectsSearch.Default instance + * using the default configuration. + * + * @param adjacencyFiles the input adjacency files. + * @param referenceSetsPaths list of path's to the reference set files. + */ + public Default(final List adjacencyFiles, final List referenceSetsPaths) + { + this.adjacencyFiles = adjacencyFiles; + this.referenceSetsPaths = referenceSetsPaths; + this.configuration = new DefaultConfiguration(); + + this.threadFactory = new ThreadFactory() + { + final AtomicInteger counter = new AtomicInteger(); + + @Override + public Thread newThread(final Runnable r) + { + return new Thread(r, "Eclipse-Store-WorkerThread-" + this.counter.getAndIncrement()); + } + }; + + for(final AdjacencyFiles channel : this.adjacencyFiles) + { + this.fileCount += channel.get().size(); + } + } + + /** + * Create a new MissingObjectsSearch.Default instance. + * + * @param adjacencyFiles the input adjacency files. + * @param referenceSetsPaths list of path's to the reference set files. + * @param configuration configuration, can be null. + */ + public Default(final List adjacencyFiles, final List referenceSetsPaths, final DefaultConfiguration configuration) + { + this.adjacencyFiles = adjacencyFiles; + this.referenceSetsPaths = referenceSetsPaths; + this.configuration = configuration; + + this.threadFactory = new ThreadFactory() + { + final AtomicInteger counter = new AtomicInteger(); + + @Override + public Thread newThread(final Runnable r) + { + return new Thread(r, "Eclipse-Store-WorkerThread-" + this.counter.getAndIncrement()); + } + }; + + for(final AdjacencyFiles channel : this.adjacencyFiles) + { + this.fileCount += channel.get().size(); + } + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public MissingObjects searchMissingEntities() + { + final LinkedBlockingQueue refSetsQueueDone = this.searchMissingObjectIDs(this.referenceSetsPaths); + + return new MissingObjects.Default(this.gatherMissingObjectIDs(refSetsQueueDone)); + } + + private Set gatherMissingObjectIDs(final LinkedBlockingQueue refSetsQueue) + { + final Set missingObjectIds = new HashSet<>(); + refSetsQueue.forEach((set) -> + { + set.load(false); + if(!set.getReferences().isEmpty()) + { + logger.info("file {} has {} missing object(s)!", set.getPath().getFileName(), set.getReferences().size()); + } + missingObjectIds.addAll(set.getReferences()); + set.unload(); + }); + + return missingObjectIds; + } + + private static Void shutdownExceptional(final ExecutorService ex, final Throwable t) { + if(ex != null && !ex.isTerminated()) + { + ex.shutdownNow(); + logger.error("Shutting down exceptionally: ", t); + throw new RuntimeException("abnormal termination", t); + } + + return null; + } + + private static Void shutdown(final ExecutorService ex) { + if(ex != null && !ex.isTerminated()) + { + ex.shutdownNow(); + } + return null; + } + + private LinkedBlockingQueue searchMissingObjectIDs(final List referenceSetsPaths) + { + final ExecutorService executor = Executors.newFixedThreadPool(this.configuration.getReduceStage_TotalThreads(), this.threadFactory); + + + final LinkedBlockingQueue refSetsQueue = new LinkedBlockingQueue<>(); + referenceSetsPaths.forEach(p -> refSetsQueue.add(new AdjacencySet(p))); + + final LinkedBlockingQueue adjacentMapsPathsQueue = new LinkedBlockingQueue<>(); + final LinkedBlockingQueue adjacencyMapsQueue = new LinkedBlockingQueue<>(this.configuration.getReduceStage_MapLoaders()); + final LinkedBlockingQueue initOutQueue = new LinkedBlockingQueue<>(this.configuration.getQUEUE_SIZE()); + final LinkedBlockingQueue adjacencySetsQueueFinished = new LinkedBlockingQueue<>(); + LinkedBlockingQueue reducerInQueue; //created later on demand + + for(final AdjacencyFiles channel : this.adjacencyFiles) + { + for(final Entry entry : channel.get().entrySet()) + { + adjacentMapsPathsQueue.add(entry.getValue()); + } + } + + final List> futures = new ArrayList<>(); + + for(int i = 0; i < this.configuration.getReduceStage_MapLoaders(); i++) + { + futures.add( + CompletableFuture + .runAsync(() -> AdjacencyDataConverter.Default.adjacencyMapLoader(adjacentMapsPathsQueue, List.of(adjacencyMapsQueue)), executor) + .exceptionally((t) -> shutdownExceptional(executor, t)) + ); + } + + for(int i = 0; i < this.configuration.getReduceStage_Initializers(); i++) + { + futures.add( + CompletableFuture + .runAsync(() -> this.stageInit(refSetsQueue, initOutQueue), executor) + .exceptionally((t) -> shutdownExceptional(executor, t)) + ); + } + + reducerInQueue = initOutQueue; + for(int i = 0; i < this.configuration.getReduceStage_SetReducers(); i++) { + + final LinkedBlockingQueue inQueue = reducerInQueue; + final LinkedBlockingQueue outQueue = new LinkedBlockingQueue<>(this.configuration.getQUEUE_SIZE()); + + + futures.add( + CompletableFuture + .runAsync(() -> this.stageReduce(adjacencyMapsQueue, inQueue, outQueue), executor) + .exceptionally((t) -> shutdownExceptional(executor, t)) + ); + + reducerInQueue = outQueue; + } + + final LinkedBlockingQueue stageEndInQueue = reducerInQueue; + futures.add( + CompletableFuture + .runAsync(() -> this.stageEnd(stageEndInQueue, refSetsQueue, adjacencySetsQueueFinished), executor) + .whenComplete((v, t) -> { + if(t!=null) { + shutdownExceptional(executor, t); + } else { + shutdown(executor); + } + }) + ); + + final CompletableFuture reduceStage = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); + + try + { + reduceStage.get(); + } + catch(InterruptedException | ExecutionException e) + { + logger.error("searchMissingObjectIDs failed: ", e); + throw new RuntimeException(e); + } + finally + { + executor.shutdownNow(); + } + + return adjacencySetsQueueFinished; + } + + private void stageInit( + final LinkedBlockingQueue queueIn, + final LinkedBlockingQueue queueOut) + { + do + { + try + { + final AdjacencySet set = queueIn.take(); + set.load(true); + queueOut.put(set); + } + catch(final InterruptedException e) + { + //stop processing + logger.debug("Init stage thread ending stopped of interruption"); + return; + } + catch(final Exception e) + { + throw new RuntimeException(e); + } + } while(true); + } + + private void stageReduce( + final LinkedBlockingQueue adjacencyMapsQueue, + final LinkedBlockingQueue queueIn, + final LinkedBlockingQueue queueOut) + { + while(true) + { + if(this.mapsTaken.getAndIncrement() < this.fileCount) + { + try + { + final AdjacencyMap map = adjacencyMapsQueue.take(); + + long counter = 0; + + while(counter < this.fileCount) + { + + final AdjacencySet set = queueIn.take(); + if(!set.isEmpty()) + { + set.reduce(map); + } + + queueOut.put(set); + counter++; + } + } + catch(final InterruptedException e) + { + //stop processing + logger.debug("Reduce stage thread stopped because of interruption"); + return; + } + catch(final Exception e) + { + throw new RuntimeException(e); + } + } + else + { + //simply forward + try + { + final AdjacencySet set = queueIn.take(); + queueOut.put(set); + } + catch(final InterruptedException e) + { + //stop processing + logger.debug("Reduce stage thread stopped because of interruption"); + return; + } + catch(final Exception e) + { + throw new RuntimeException(e); + } + } + } + } + + private void stageEnd(final LinkedBlockingQueue queueIn, + final LinkedBlockingQueue queueOut, + final LinkedBlockingQueue queueDone) + { + long processedFiles = 0; + final List tmp = new ArrayList<>(); + + do + { + try + { + final AdjacencySet set = queueIn.take(); + + set.store(); + set.unload(); + + tmp.add(set); + + //every time a set reaches this point it was reduced by + //n reduces stages. We are done as soon as very file was + //reduced by every file, which results in a total of + //files^2 reduce operations. + processedFiles += this.configuration.getReduceStage_SetReducers(); + + if(tmp.size() == this.fileCount) + { + logger.debug("Processed {} of {}", processedFiles, this.fileCount * this.fileCount); + + if(processedFiles >= (this.fileCount * this.fileCount)) + { + if(queueOut.isEmpty()) + { + queueDone.addAll(tmp); + return; + } else + throw new RuntimeException("Something went wrong, there are more items to be processed then expected."); + } + else + { + queueOut.addAll(tmp); + tmp.clear(); + } + } + + } + catch(final InterruptedException e) + { + //stop processing + logger.debug("End stage thread stopped because of interruption"); + return; + } + catch(final Exception e) + { + throw new RuntimeException(e); + } + + } + while(true); + } + + } + +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ObjectParents.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ObjectParents.java new file mode 100644 index 00000000..4322fa51 --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ObjectParents.java @@ -0,0 +1,69 @@ +package org.eclipse.store.storage.analysis; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.util.Map; + +/** + * Interface defining the result of reverse object search. + */ +public interface ObjectParents +{ + /** + * Get the chain of references from root to the object + * associated with the supplied id. + * The returned array starts with the root object id and + * ends with the missing object id. + * + * @param objectID as long. + * @return an array of object IDs. + */ + long[] getParents(long objectID); + + /** + * Provides the result of the reverse object search. + */ + public class Default implements ObjectParents + { + private final Map parents; + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + public Default(final Map parents) + { + super(); + this.parents = parents; + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + //////////// + + @Override + public long[] getParents(final long objectID) + { + return this.parents.get(objectID); + } + + @Override + public String toString() + { + return "Default [parents=" + this.parents + "]"; + } + + } +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ReverseObjectSearch.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ReverseObjectSearch.java new file mode 100644 index 00000000..2ca8f2f0 --- /dev/null +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ReverseObjectSearch.java @@ -0,0 +1,276 @@ +package org.eclipse.store.storage.analysis; + +/*- + * #%L + * EclipseStore Storage + * %% + * Copyright (C) 2023 - 2025 MicroStream Software + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import java.nio.file.Path; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.eclipse.serializer.util.logging.Logging; +import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; +import org.slf4j.Logger; + +public interface ReverseObjectSearch +{ + /** + * Search for missing objects by id. + * + * @param objectIDs a set of id to search + * @return ObjectParents instance providing the search results. + */ + public ObjectParents searchObjectIDs(final Set objectIDs); + + /** + * Create a new default instance of the ReverseObjectSearch. + * + * @param adjacencyFiles to be processed. + * @return a new ReverseObjectSearch instance. + */ + public static ReverseObjectSearch New(final List adjacencyFiles) + { + return new Default(adjacencyFiles); + } + + public class Default implements ReverseObjectSearch + { + private final static Logger logger = Logging.getLogger(ReverseObjectSearch.class); + + private final List adjacencyFiles; + private final LinkedList reverseAdjacencyMaps; + + + /////////////////////////////////////////////////////////////////////////// + // constructors // + ///////////////// + + /** + * Create a new instance of the ReverseObjectSearch. + * + * @param adjacencyFiles to be processed. + */ + public Default(final List adjacencyFiles) + { + super(); + this.adjacencyFiles = adjacencyFiles; + this.reverseAdjacencyMaps = this.buildReverseAdjacencyMaps(); + } + + /////////////////////////////////////////////////////////////////////////// + // methods // + /////////// + + @Override + public ObjectParents searchObjectIDs(final Set objectIDs) + { + while(this.hasOverlapping(this.reverseAdjacencyMaps)) + { + this.sort(this.reverseAdjacencyMaps); + Collections.sort(this.reverseAdjacencyMaps); + } + + return this.search(objectIDs); + } + + private boolean hasOverlapping(final LinkedList list) + { + for(final ComparableAdjacencyMap mapA : list) + { + for(final ComparableAdjacencyMap mapB : list) + { + if(mapA.isEmpty() || mapB.isEmpty()) { + continue; + } + + final boolean overlapping = mapA.intersect(mapB); + logger.trace("overlap test {} > {} : {}", mapA.path.getFileName(), mapB.path.getFileName(), overlapping); + + if(mapA != mapB) + { + if(overlapping) + return true; + } + } + } + return false; + } + + private ObjectParents.Default search(final Set objectIDs) + { + final Map foundParents = new TreeMap<>(); + + final TreeSet next = new TreeSet<>(objectIDs); + + while(!next.isEmpty()) + { + Long currentID = next.pollFirst(); + + logger.debug("searching processing id: {} ", currentID); + + final ComparableAdjacencyMap map = this.getMapFor(currentID); + if(map != null) + { + if(map.map == null) + { + map.load(); + } + + while(currentID != null) + { + final long[] parents = map.map.get(currentID); + if(parents == null) + { + logger.trace("no parents for id {} found in map {} !", currentID, map); + } + else + { + foundParents.put(currentID, parents); + + for(final long parentID : parents) + { + if(foundParents.get(parentID) == null) + { + //if parent object id not found search for it in next cycle. + next.add(parentID); + } + } + } + + if(!next.isEmpty()) + { + currentID = next.first(); + } + + if(!next.removeIf(map::inRange)) + { + currentID = null; + } + } + + map.unload(); + + } + else + { + logger.trace("ID {} not found in map ranges!", currentID); + } + } + + return new ObjectParents.Default(foundParents); + } + + private ComparableAdjacencyMap getMapFor(final long id) + { + for(final ComparableAdjacencyMap map : this.reverseAdjacencyMaps) + { + if(map.inRange(id)) + return map; + } + return null; + } + + private void sort(final LinkedList list) + { + long counter = 0; + + for(final ComparableAdjacencyMap mapA : list) + { + if(mapA.isEmpty()) { + continue; + } + mapA.load(); + + for(final ComparableAdjacencyMap mapB : list) + { + if(mapA == mapB) { + continue; + } + + if(mapB.isEmpty()) { + continue; + } + if(mapA.intersect(mapB)) + { + mapB.load(); + + this.mergeMaps(mapA, mapB); + + mapA.update(); + mapB.update(); + + if(mapB.updated()) + { + mapB.serialize(); + logger.trace("serialized {} ", mapB.path.getFileName()); + } + mapB.unload(); + } + + } + if(mapA.updated()) + { + mapA.serialize(); + } + mapA.unload(); + + logger.debug("Progress: {} of {}", ++counter, list.size()); + } + } + + private void mergeMaps(final ComparableAdjacencyMap mapA, final ComparableAdjacencyMap mapB) + { + logger.trace("merging {} with {}", mapA, mapB); + + final TreeMap map1 = mapA.map; + final TreeMap map2 = mapB.map; + + map1.putAll(map2); + map2.clear(); + + for(int i = 0; i < map1.size() / 2; i++) + { + final var e = map1.lastEntry(); + map2.put(e.getKey(), e.getValue()); + map1.remove(e.getKey()); + } + + mapA.update(); + mapB.update(); + } + + + private LinkedList buildReverseAdjacencyMaps() + { + final LinkedList backReferenceMaps = new LinkedList<>(); + + for(final AdjacencyFiles channel : this.adjacencyFiles) + { + for(final Entry entry : channel.get().entrySet()) + { + final Path path = AdjacencyDataConverter.Default.derivePath(entry.getValue(), ".brf"); + + backReferenceMaps.add(new ComparableAdjacencyMap(path)); + } + } + + return backReferenceMaps; + } + } +} diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageAdjacencyDataExporter.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageAdjacencyDataExporter.java index 65ae6fb4..200851c7 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageAdjacencyDataExporter.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageAdjacencyDataExporter.java @@ -190,9 +190,9 @@ private void serialize( final long keyCount, final long refCount) { - long estimatedbufferSize = ( - (XMemory.byteSize_long() + XMemory.byteSize_int()) * keyCount) + - (refCount * XMemory.byteSizeReference()); + long estimatedbufferSize = + ((XMemory.byteSize_long() + XMemory.byteSize_int()) * keyCount) + + (refCount * XMemory.byteSize_long()); ByteBuffer buffer = ByteBuffer.allocate((int)estimatedbufferSize); @@ -202,7 +202,14 @@ private void serialize( buffer.putInt(entry.getValue().length); for (long l : entry.getValue()) { - buffer.putLong(l); + try + { + buffer.putLong(l); + } + catch(Exception e) + { + throw new RuntimeException(e); + } } } buffer.flip(); From 989d0e3ad476cab0095a39bc8c1f13e91916a7ee Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Thu, 26 Jun 2025 15:28:54 +0200 Subject: [PATCH 12/14] removed dev syso, fixed export task result initialization --- .../storage/types/StorageConnection.java | 12 ++------- ...StorageRequestTaskExportAdjacencyData.java | 27 +++---------------- 2 files changed, 6 insertions(+), 33 deletions(-) diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageConnection.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageConnection.java index dbc5536a..58b10834 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageConnection.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageConnection.java @@ -221,11 +221,8 @@ public default void issueFullBackup(final ADirectory targetDirectory) PersistenceTypeDictionaryFileHandler.New(targetDirectory) ) ); - } - else - { + } else throw new StorageExceptionBackupFullBackupTargetNotEmpty(targetDirectory); - } } /** @@ -601,12 +598,7 @@ public List exportAdjacencyData(final Path workingDir) { final PersistenceObjectRegistry registry = this.persistenceManager().objectRegistry(); final PersistenceRootsView roots = this.persistenceManager().viewRoots(); - - roots.iterateEntries((s,o) -> { - final long id = registry.lookupObjectId(o); - System.out.println(s + " " + o + ",id: " + id); - }); - + final Object defaultRoot = roots.rootReference().get(); if(defaultRoot != null) { diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskExportAdjacencyData.java b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskExportAdjacencyData.java index 07b1def4..823686ef 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskExportAdjacencyData.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/types/StorageRequestTaskExportAdjacencyData.java @@ -2,22 +2,6 @@ import java.nio.file.Files; import java.nio.file.Path; - -/*- - * #%L - * EclipseStore Storage - * %% - * Copyright (C) 2023 - 2025 MicroStream Software - * %% - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * #L% - */ - -import java.util.Arrays; import java.util.List; import org.eclipse.store.storage.exceptions.StorageException; @@ -41,7 +25,7 @@ public final class Default extends StorageChannelSynchronizingTask.AbstractCompletingTask implements StorageRequestTaskExportAdjacencyData, StorageChannelTask { - private final List channelResults; + private final AdjacencyFiles[] channelResults; private final Path exportDirectory; /////////////////////////////////////////////////////////////////////////// @@ -51,7 +35,7 @@ public final class Default public Default(final long timestamp, final int channelCount, final StorageOperationController controller, final Path exportDirectory) { super(timestamp, channelCount, controller); - this.channelResults = Arrays.asList(null, null, null, null); + this.channelResults = new AdjacencyFiles[channelCount]; this.exportDirectory = this.ensureDirectory(exportDirectory); } @@ -63,8 +47,7 @@ public Default(final long timestamp, final int channelCount, final StorageOperat protected final Void internalProcessBy(final StorageChannel channel) { final AdjacencyFiles result = channel.collectAdjacencyData(this.exportDirectory); - this.channelResults.set(channel.channelIndex(), result); - + this.channelResults[channel.channelIndex()]= result; return null; } @@ -77,15 +60,13 @@ protected void postCompletionSuccess(final StorageChannel channel, final Void re @Override public final List result() { - return this.channelResults; + return List.of(this.channelResults); } private Path ensureDirectory(final Path exportDirectory) { if(!Files.exists(exportDirectory)) - { throw new StorageException("Directory not found: " + exportDirectory); - } return exportDirectory; } From 7d2db7ff225db86fff93341583e0f1ba0631e2c9 Mon Sep 17 00:00:00 2001 From: Zdenek Jonas Date: Mon, 30 Jun 2025 12:13:42 +0200 Subject: [PATCH 13/14] Revert "chore: update build configurations to use JDK 17 (#417)" (#418) This reverts commit f75b9f5c281671b8ee9356b71d115d8907ee34b0. --- .github/workflows/maven_build.yml | 12 ++++ .github/workflows/maven_build_win.yml | 13 ++++- .github/workflows/maven_converter.yml | 5 +- .github/workflows/maven_deploy_snapshot.yml | 36 +++++++++++- .../workflows/maven_deploy_snapshot_dev.yml | 57 ++++++++++++++++++- integrations/itest/pom.xml | 1 - integrations/pom.xml | 37 ++++++++++-- integrations/spring-boot3-console/pom.xml | 1 - integrations/spring-boot3/pom.xml | 1 - pom.xml | 3 +- .../client-app-standalone-assembly/pom.xml | 1 - storage/rest/client-app/pom.xml | 1 - storage/rest/pom.xml | 36 +++++++++++- storage/rest/service-springboot/pom.xml | 1 - 14 files changed, 186 insertions(+), 19 deletions(-) diff --git a/.github/workflows/maven_build.yml b/.github/workflows/maven_build.yml index 70a16143..ac7813b4 100644 --- a/.github/workflows/maven_build.yml +++ b/.github/workflows/maven_build.yml @@ -14,6 +14,18 @@ jobs: build: runs-on: ubuntu-latest steps: + #Build with java 11 + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: 'maven' + - name: Build with Maven + run: mvn -P examples -B clean package --file pom.xml -U + + #Build with java 17 - uses: actions/checkout@v3 - name: Set up JDK 17 uses: actions/setup-java@v3 diff --git a/.github/workflows/maven_build_win.yml b/.github/workflows/maven_build_win.yml index 912eb9f4..2411e0e7 100644 --- a/.github/workflows/maven_build_win.yml +++ b/.github/workflows/maven_build_win.yml @@ -14,6 +14,18 @@ jobs: build: runs-on: windows-latest steps: + #Build with java 11 + - uses: actions/checkout@v3 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: 'maven' + - name: Build with Maven + run: mvn -P examples -B clean package --file pom.xml -U + + #Build with java 17 - uses: actions/checkout@v3 - name: Set up JDK 17 uses: actions/setup-java@v3 @@ -23,4 +35,3 @@ jobs: cache: 'maven' - name: Build with Maven run: mvn -P examples -B clean package --file pom.xml -U - diff --git a/.github/workflows/maven_converter.yml b/.github/workflows/maven_converter.yml index 3c056ae2..104fa5c8 100644 --- a/.github/workflows/maven_converter.yml +++ b/.github/workflows/maven_converter.yml @@ -14,11 +14,12 @@ jobs: build: runs-on: ubuntu-latest steps: + #Build with java 11 - uses: actions/checkout@v3 - - name: Set up JDK 17 + - name: Set up JDK 11 uses: actions/setup-java@v3 with: - java-version: '17' + java-version: '11' distribution: 'adopt' - name: Build with Maven run: mvn -B -Pconverter-standalone -pl storage/embedded-tools/storage-converter -am clean package --file pom.xml -U diff --git a/.github/workflows/maven_deploy_snapshot.yml b/.github/workflows/maven_deploy_snapshot.yml index 472ee0ee..95584ce1 100644 --- a/.github/workflows/maven_deploy_snapshot.yml +++ b/.github/workflows/maven_deploy_snapshot.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Java for publishing to Maven Central Snapshot Repository uses: actions/setup-java@v4 with: - java-version: '17' + java-version: '11' distribution: 'temurin' cache: 'maven' server-id: ossrh @@ -32,3 +32,37 @@ jobs: MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} + #java 17 build + publish_java17: + if: github.repository == 'eclipse-store/store' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Java 17 for publishing to Maven Central Repository + uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - name: Build with java 17 + run: | + mvn -P production -pl integrations/spring-boot3 clean install -am -B + mvn -P production -pl integrations/spring-boot3-console clean install -am -B + mvn -P production -pl storage/rest/client-app clean install -am -B + mvn -P production -pl storage/rest/client-app-standalone-assembly clean install -am -B + mvn -P production -pl storage/rest/service-springboot clean install -am -B + - name: Deploy module build with java 17 + run: | + mvn -Pdeploy -Pproduction -pl integrations/spring-boot3 clean deploy + mvn -Pdeploy -Pproduction -pl integrations/spring-boot3-console clean deploy + mvn -Pdeploy -Pproduction -pl storage/rest/client-app clean deploy + mvn -Pdeploy -Pproduction -pl storage/rest/client-app-standalone-assembly clean deploy + mvn -Pdeploy -Pproduction -pl storage/rest/service-springboot clean deploy + env: + MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} + MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} diff --git a/.github/workflows/maven_deploy_snapshot_dev.yml b/.github/workflows/maven_deploy_snapshot_dev.yml index 9bb68b13..83a97c8c 100644 --- a/.github/workflows/maven_deploy_snapshot_dev.yml +++ b/.github/workflows/maven_deploy_snapshot_dev.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Java for publishing to Maven Central Snapshot Repository uses: actions/setup-java@v4 with: - java-version: '17' + java-version: '11' distribution: 'temurin' cache: 'maven' server-id: ossrh @@ -52,3 +52,58 @@ jobs: MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} + + #java 17 build + publish_java17: + if: github.repository == 'eclipse-store/store' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Java for publishing to Maven Central Snapshot Repository + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - name: Prepare suffix + run: | + suffix=$(echo -n "${GITHUB_REF#refs/heads/}" | tr '/' '_' | cut -c1-10)-$(echo -n "${GITHUB_REF#refs/heads/}" | md5sum | cut -c1-10) + echo "Suffix: $suffix" + echo "SUFFIX=$suffix" >> $GITHUB_ENV + - name: Update project version java 17 + run: | + currentVersion=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec) + currentVersionWithoutSnapshot=${currentVersion%-SNAPSHOT} + newVersion="${currentVersionWithoutSnapshot}-$SUFFIX-SNAPSHOT" + mvn versions:set -DnewVersion=$newVersion --batch-mode + BRANCH_NAME=${{ github.ref }} + REPO_OWNER="eclipse-serializer" + REPO_NAME="serializer" + RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/$REPO_OWNER/$REPO_NAME/branches/$BRANCH_NAME) + if [ $RESPONSE -eq 200 ]; then + mvn versions:set-property -Dproperty=eclipse.serializer.version -DnewVersion=$newVersion + else + echo "Branch does not exist in serializer repository, skipping serializer version change" + fi + - name: Make a snapshot java 17 + run: | + mvn -P production -pl integrations/spring-boot3 clean install -am -B + mvn -P production -pl integrations/spring-boot3-console clean install -am -B + mvn -P production -pl storage/rest/client-app clean install -am -B + mvn -P production -pl storage/rest/client-app-standalone-assembly clean install -am -B + mvn -P production -pl storage/rest/service-springboot clean install -am -B + - name: Deploy module build with java 17 + run: | + mvn -Pdeploy -Pproduction -pl integrations/spring-boot3 clean deploy + mvn -Pdeploy -Pproduction -pl integrations/spring-boot3-console clean deploy + mvn -Pdeploy -Pproduction -pl storage/rest/client-app clean deploy + mvn -Pdeploy -Pproduction -pl storage/rest/client-app-standalone-assembly clean deploy + mvn -Pdeploy -Pproduction -pl storage/rest/service-springboot clean deploy + env: + MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.ORG_GPG_PASSPHRASE }} + MAVEN_GPG_KEY: ${{ secrets.ORG_GPG_PRIVATE_KEY }} diff --git a/integrations/itest/pom.xml b/integrations/itest/pom.xml index 7e705d56..17fd46b4 100644 --- a/integrations/itest/pom.xml +++ b/integrations/itest/pom.xml @@ -17,7 +17,6 @@ 3.2.2 - 17 diff --git a/integrations/pom.xml b/integrations/pom.xml index 7a10ca96..3d0c9d29 100644 --- a/integrations/pom.xml +++ b/integrations/pom.xml @@ -18,10 +18,39 @@ cdi4 - spring-boot3 - spring-boot3-console - - itest + + + from_java_17 + + [17,) + + + spring-boot3 + spring-boot3-console + + itest + + + 17 + 17 + 17 + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + 17 + + + + + + + + diff --git a/integrations/spring-boot3-console/pom.xml b/integrations/spring-boot3-console/pom.xml index 598eace9..4cb3fe32 100644 --- a/integrations/spring-boot3-console/pom.xml +++ b/integrations/spring-boot3-console/pom.xml @@ -17,7 +17,6 @@ 3.4.5 - 17 diff --git a/integrations/spring-boot3/pom.xml b/integrations/spring-boot3/pom.xml index dc9acd53..fa15453d 100644 --- a/integrations/spring-boot3/pom.xml +++ b/integrations/spring-boot3/pom.xml @@ -17,7 +17,6 @@ 3.4.5 - 17 diff --git a/pom.xml b/pom.xml index 9b0883d9..9dd20d1e 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,8 @@ UTF-8 - 11 + 11 + 11 javac 3.8.1 11 diff --git a/storage/rest/client-app-standalone-assembly/pom.xml b/storage/rest/client-app-standalone-assembly/pom.xml index 420d74a1..ec465e6a 100644 --- a/storage/rest/client-app-standalone-assembly/pom.xml +++ b/storage/rest/client-app-standalone-assembly/pom.xml @@ -21,7 +21,6 @@ 24.2.6 3.2.0 - 17 diff --git a/storage/rest/client-app/pom.xml b/storage/rest/client-app/pom.xml index ad9790dc..70280efd 100644 --- a/storage/rest/client-app/pom.xml +++ b/storage/rest/client-app/pom.xml @@ -22,7 +22,6 @@ 24.2.6 3.2.0 - 17 diff --git a/storage/rest/pom.xml b/storage/rest/pom.xml index 0ad10a33..c2fafbb9 100644 --- a/storage/rest/pom.xml +++ b/storage/rest/pom.xml @@ -26,9 +26,39 @@ client-jersey service service-sparkjava - service-springboot - client-app - client-app-standalone-assembly + + + from_java_17 + + [17,) + + + 17 + 17 + 17 + + + service-springboot + client-app + client-app-standalone-assembly + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + 17 + + + + + + + + + diff --git a/storage/rest/service-springboot/pom.xml b/storage/rest/service-springboot/pom.xml index 10dd5cee..00ae0150 100644 --- a/storage/rest/service-springboot/pom.xml +++ b/storage/rest/service-springboot/pom.xml @@ -21,7 +21,6 @@ 3.2.2 - 17 From 7112001b66303695891c0264cae4959d62665a71 Mon Sep 17 00:00:00 2001 From: hg-ms <53219833+hg-ms@users.noreply.github.com> Date: Wed, 2 Jul 2025 08:28:07 +0200 Subject: [PATCH 14/14] Missing objects search improvements 01072025 (#419) * Fixed thread ending condition * improved java doc * Added required data to constructor --- .../analysis/AdjacencyDataConverter.java | 4 +-- .../storage/analysis/ReverseObjectSearch.java | 30 ++++++++----------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyDataConverter.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyDataConverter.java index 4c5f4f0a..0eea0499 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyDataConverter.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/AdjacencyDataConverter.java @@ -409,11 +409,11 @@ protected static void adjacencyMapLoader( final LinkedBlockingQueue adjacencyMapsPathsQueue, final List> outputQueues) { - while(!adjacencyMapsPathsQueue.isEmpty()) + Path path; + while((path = adjacencyMapsPathsQueue.poll()) != null) { try { - final Path path = adjacencyMapsPathsQueue.take(); final TreeMap refMap = AdjacencyMap.deserializeReferenceMap(path); for(final LinkedBlockingQueue queue : outputQueues) diff --git a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ReverseObjectSearch.java b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ReverseObjectSearch.java index 2ca8f2f0..6eaddab6 100644 --- a/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ReverseObjectSearch.java +++ b/storage/storage/src/main/java/org/eclipse/store/storage/analysis/ReverseObjectSearch.java @@ -19,19 +19,19 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import org.eclipse.serializer.util.logging.Logging; +import org.eclipse.store.storage.analysis.AdjacencyDataConverter.ConvertedAdjacencyFiles; import org.eclipse.store.storage.types.StorageAdjacencyDataExporter.AdjacencyFiles; import org.slf4j.Logger; public interface ReverseObjectSearch { /** - * Search for missing objects by id. + * Search for parent objects by id. * * @param objectIDs a set of id to search * @return ObjectParents instance providing the search results. @@ -42,11 +42,12 @@ public interface ReverseObjectSearch * Create a new default instance of the ReverseObjectSearch. * * @param adjacencyFiles to be processed. + * @param convertedAdjacencyFiles required convertedAdjacencyFiles. * @return a new ReverseObjectSearch instance. */ - public static ReverseObjectSearch New(final List adjacencyFiles) + public static ReverseObjectSearch New(final List adjacencyFiles, ConvertedAdjacencyFiles convertedAdjacencyFiles) { - return new Default(adjacencyFiles); + return new Default(adjacencyFiles, convertedAdjacencyFiles); } public class Default implements ReverseObjectSearch @@ -65,12 +66,13 @@ public class Default implements ReverseObjectSearch * Create a new instance of the ReverseObjectSearch. * * @param adjacencyFiles to be processed. + * @param convertedAdjacencyFiles required convertedAdjacencyFiles. */ - public Default(final List adjacencyFiles) + public Default(final List adjacencyFiles, ConvertedAdjacencyFiles convertedAdjacencyFiles) { super(); this.adjacencyFiles = adjacencyFiles; - this.reverseAdjacencyMaps = this.buildReverseAdjacencyMaps(); + this.reverseAdjacencyMaps = this.buildReverseAdjacencyMaps(convertedAdjacencyFiles.getReverseReferenceMaps()); } /////////////////////////////////////////////////////////////////////////// @@ -256,19 +258,13 @@ private void mergeMaps(final ComparableAdjacencyMap mapA, final ComparableAdjace } - private LinkedList buildReverseAdjacencyMaps() + private LinkedList buildReverseAdjacencyMaps(List reverseReverenceMapPaths) { final LinkedList backReferenceMaps = new LinkedList<>(); - - for(final AdjacencyFiles channel : this.adjacencyFiles) - { - for(final Entry entry : channel.get().entrySet()) - { - final Path path = AdjacencyDataConverter.Default.derivePath(entry.getValue(), ".brf"); - - backReferenceMaps.add(new ComparableAdjacencyMap(path)); - } - } + + reverseReverenceMapPaths.forEach( + path -> backReferenceMaps.add(new ComparableAdjacencyMap(path)) + ); return backReferenceMaps; }