From 015f49b4999acbf63460f55ad16c60fbea68374d Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 13:30:32 -0700 Subject: [PATCH 01/13] refactor(service): split CollectionDAO & EntityRepository; fix lineage/usage/isBot bugs Decompose the two backend god-classes into cohesive, single-responsibility collaborators (behavior-preserving) and fix several DAO-layer perf/correctness bugs. CollectionDAO (15,040 -> 433 lines): split into 17 same-package domain aggregator interfaces it `extends` (composite-interface-inheritance, zero call-site change): RdfInfraDAOs, SearchReindexDAOs, AiGovernanceDAOs, FeedDAOs, ClassificationTagDAOs, TimeSeriesDAOs, ActivityAuditDAOs, GovernanceDAOs, EventSubscriptionDAOs, KnowledgeAssetDAOs, SystemTokenDAOs, DataAssetServiceDAOs, EntityDataDAOs, AccessControlDAOs, WorkflowDocStoreDAOs, OAuthDAOs, CoreRelationshipDAOs. Added CollectionDAOCompositionTest guarding the JDBI wiring of inherited accessors. EntityRepository (12,797 -> ~9,950 lines): extracted 7 collaborators with delegators preserving the public/protected API for 81 subclasses: EntityTaskWorkflows, EntityCacheLoaders, EntityCaches, EntityCacheInvalidator, InheritanceParentCache, BulkFieldFetcher, BulkImportService. Perf/correctness fixes: - Lineage findTo/FromPipeline operator-precedence bug (restores the relation filter) - entity_usage(entityType,usageDate) index for UsageDAO.computePercentile (2.0.0 migration) - Postgres isBot generated column read $.deleted instead of $.isBot (2.0.0 migration); bots were counted as daily-active users on Postgres - getMaxLastActivityTime / user-list isBot filters use the indexed columns - deleted dead, malformed EntityExtensionDAO.update Full openmetadata-service test suite passes (5,800 tests, 0 failures). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../native/2.0.0/mysql/schemaChanges.sql | 23 + .../native/2.0.0/postgres/schemaChanges.sql | 21 + .../service/OpenMetadataApplication.java | 5 +- .../apps/bundles/rdf/RdfBatchProcessor.java | 2 +- .../service/apps/bundles/rdf/RdfIndexApp.java | 2 +- .../DistributedRdfIndexCoordinator.java | 10 +- .../DistributedSearchIndexCoordinator.java | 16 +- .../distributed/JobRecoveryManager.java | 2 +- .../searchIndex/stats/JobStatsManager.java | 4 +- .../service/audit/AuditLogConsumer.java | 2 +- .../service/cache/BundleWarmupBatcher.java | 4 +- .../service/cache/CacheBundle.java | 4 +- .../service/cache/Invalidatable.java | 2 +- .../service/jdbi3/AccessControlDAOs.java | 1142 ++ .../service/jdbi3/ActivityAuditDAOs.java | 509 + .../service/jdbi3/AiGovernanceDAOs.java | 343 + .../service/jdbi3/BulkFieldFetcher.java | 1271 ++ .../service/jdbi3/BulkImportService.java | 1078 ++ .../jdbi3/ClassificationRepository.java | 14 +- .../service/jdbi3/ClassificationTagDAOs.java | 1187 ++ .../service/jdbi3/CollectionDAO.java | 15302 +--------------- .../service/jdbi3/ContainerRepository.java | 8 +- .../service/jdbi3/CoreRelationshipDAOs.java | 1583 ++ .../service/jdbi3/DataAssetServiceDAOs.java | 1052 ++ .../service/jdbi3/DataProductRepository.java | 7 +- .../service/jdbi3/DomainRepository.java | 16 +- .../service/jdbi3/EntityCacheInvalidator.java | 475 + .../service/jdbi3/EntityCacheLoaders.java | 199 + .../service/jdbi3/EntityCaches.java | 156 + .../service/jdbi3/EntityDataDAOs.java | 1049 ++ .../jdbi3/EntityRelationshipRepository.java | 2 +- .../service/jdbi3/EntityRepository.java | 3163 +--- .../service/jdbi3/EntityTaskWorkflows.java | 108 + .../service/jdbi3/EventSubscriptionDAOs.java | 271 + .../openmetadata/service/jdbi3/FeedDAOs.java | 2014 ++ .../service/jdbi3/GlossaryRepository.java | 13 +- .../service/jdbi3/GlossaryTermRepository.java | 24 +- .../service/jdbi3/GovernanceDAOs.java | 214 + .../service/jdbi3/InheritanceParentCache.java | 151 + .../service/jdbi3/KnowledgeAssetDAOs.java | 382 + .../service/jdbi3/LineageRepository.java | 2 +- .../openmetadata/service/jdbi3/OAuthDAOs.java | 251 + .../service/jdbi3/PersonaRepository.java | 14 +- .../service/jdbi3/RdfInfraDAOs.java | 766 + .../service/jdbi3/SearchReindexDAOs.java | 1599 ++ .../jdbi3/ServiceEntityRepository.java | 3 +- .../service/jdbi3/SystemRepository.java | 2 +- .../service/jdbi3/SystemTokenDAOs.java | 442 + .../service/jdbi3/TableRepository.java | 5 +- .../service/jdbi3/TagRepository.java | 13 +- .../service/jdbi3/TeamRepository.java | 163 +- .../service/jdbi3/TimeSeriesDAOs.java | 1880 ++ .../service/jdbi3/UserRepository.java | 47 +- .../jdbi3/WorkflowDefinitionRepository.java | 6 +- .../service/jdbi3/WorkflowDocStoreDAOs.java | 686 + .../search/SearchReindexResource.java | 2 +- .../resources/system/SystemResource.java | 5 +- .../search/SearchIndexRetryWorker.java | 2 +- .../service/search/SearchRepository.java | 2 +- .../PolicyConditionUpdater.java | 3 +- .../service/socket/WebSocketManager.java | 2 +- .../openmetadata/service/util/EntityUtil.java | 10 +- .../bundles/rdf/RdfBatchProcessorTest.java | 2 +- .../apps/bundles/rdf/RdfIndexAppTest.java | 4 +- .../DistributedRdfIndexCoordinatorTest.java | 8 +- .../IndexingFailureRecorderTest.java | 4 +- .../searchIndex/SearchIndexAppTest.java | 4 +- ...DistributedSearchIndexCoordinatorTest.java | 14 +- .../DistributedSearchIndexExecutorTest.java | 4 +- .../distributed/JobRecoveryManagerTest.java | 12 +- .../JobRecoveryOrphanDetectionTest.java | 6 +- .../stats/StageStatsTrackerTest.java | 2 +- .../service/audit/AuditLogConsumerTest.java | 4 +- .../jdbi3/CollectionDAOCompositionTest.java | 96 + .../jdbi3/EntityRepositoryBulkFieldsTest.java | 2 +- .../EntityRepositoryCertificationTest.java | 2 +- ...temRepositoryEmbeddingsValidationTest.java | 2 +- .../SystemRepositoryMissingIndexesTest.java | 2 +- .../CachedPermissionEvaluationTest.java | 18 +- .../policyevaluator/RuleEvaluatorTest.java | 56 +- .../policyevaluator/SubjectCacheTest.java | 20 +- .../policyevaluator/SubjectContextTest.java | 32 +- .../TaskResourceContextTest.java | 8 +- .../service/util/EntityUtilTest.java | 4 +- 84 files changed, 19757 insertions(+), 18284 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AccessControlDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ActivityAuditDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AiGovernanceDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BulkFieldFetcher.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BulkImportService.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CoreRelationshipDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataAssetServiceDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCacheInvalidator.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCacheLoaders.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCaches.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDataDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTaskWorkflows.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/InheritanceParentCache.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KnowledgeAssetDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/OAuthDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/RdfInfraDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchReindexDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemTokenDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDocStoreDAOs.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/CollectionDAOCompositionTest.java diff --git a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql index e49858731edc..77ca00635852 100644 --- a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql @@ -377,3 +377,26 @@ CREATE TABLE IF NOT EXISTS `user_session` ( KEY `user_session_idle_expiry` (`status`,`idleExpiresAt`), KEY `user_session_prune` (`status`,`updatedAt`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- Perf: UsageDAO.computePercentile runs four correlated COUNT(*) subqueries that each +-- filter entity_usage on (entityType, usageDate). The only existing index is +-- UNIQUE (id, usageDate), which is unusable for that predicate, so every run full-scans +-- the table once per subquery. A composite (entityType, usageDate) index turns the +-- percentile subqueries into range scans. MySQL has no `ADD KEY IF NOT EXISTS`, so guard +-- via information_schema. +SET @ddl = ( + SELECT IF( + EXISTS ( + SELECT 1 + FROM information_schema.statistics + WHERE table_schema = DATABASE() + AND table_name = 'entity_usage' + AND index_name = 'idx_entity_usage_entitytype_usagedate' + ), + 'SELECT 1', + 'ALTER TABLE entity_usage ADD KEY idx_entity_usage_entitytype_usagedate (entityType, usageDate)' + ) +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql index 6e31b0453e6c..7585c90eadde 100644 --- a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql @@ -329,3 +329,24 @@ CREATE INDEX IF NOT EXISTS user_session_user_status_idx ON user_session USING bt CREATE INDEX IF NOT EXISTS user_session_expiry_idx ON user_session USING btree (status, expiresat); CREATE INDEX IF NOT EXISTS user_session_idle_expiry_idx ON user_session USING btree (status, idleexpiresat); CREATE INDEX IF NOT EXISTS user_session_prune_idx ON user_session USING btree (status, updatedat); + +-- Perf: UsageDAO.computePercentile runs four correlated COUNT(*) subqueries that each +-- filter entity_usage on (entityType, usageDate). The only existing index is +-- UNIQUE (id, usageDate), which is unusable for that predicate, so every run sequential-scans +-- the table once per subquery. A composite (entityType, usageDate) index turns the +-- percentile subqueries into range scans. +CREATE INDEX IF NOT EXISTS idx_entity_usage_entitytype_usagedate + ON entity_usage (entityType, usageDate); + +-- Correctness: migration 1.6.3 defined the Postgres isBot generated column as +-- (json ->> 'deleted')::boolean instead of (json ->> 'isBot'), so on Postgres isBot has +-- always mirrored `deleted` rather than the real bot flag. countDailyActiveUsers (and any +-- isBot column filter) was therefore wrong on Postgres. Postgres cannot alter a generated +-- column's expression in place, so backfill any rows missing $.isBot, drop the column +-- (this also drops idx_isBot) and recreate it reading the correct path. +UPDATE user_entity SET json = jsonb_set(json, '{isBot}', 'false'::jsonb, true) + WHERE (json ->> 'isBot') IS NULL; +ALTER TABLE user_entity DROP COLUMN IF EXISTS isBot; +ALTER TABLE user_entity + ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS ((json ->> 'isBot')::boolean) STORED NOT NULL; +CREATE INDEX IF NOT EXISTS idx_isBot ON user_entity (isBot); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 79319c378e52..cda7fa87bf11 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -108,6 +108,7 @@ import org.openmetadata.service.governance.workflows.WorkflowHandler; import org.openmetadata.service.jdbi3.BulkExecutor; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.EntityCaches; import org.openmetadata.service.jdbi3.EntityRelationshipRepository; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.MigrationDAO; @@ -1230,8 +1231,8 @@ public void start() { @Override public void stop() throws InterruptedException, SchedulerException { - LOG.info("Cache with Id Stats {}", EntityRepository.CACHE_WITH_ID.stats()); - LOG.info("Cache with name Stats {}", EntityRepository.CACHE_WITH_NAME.stats()); + LOG.info("Cache with Id Stats {}", EntityCaches.CACHE_WITH_ID.stats()); + LOG.info("Cache with name Stats {}", EntityCaches.CACHE_WITH_NAME.stats()); EventPubSub.shutdown(); EventSubscriptionScheduler.shutDown(); AsyncService.getInstance().shutdown(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/RdfBatchProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/RdfBatchProcessor.java index 3be5316d0912..a60c4592ed55 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/RdfBatchProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/RdfBatchProcessor.java @@ -29,7 +29,7 @@ import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipObject; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipObject; import org.openmetadata.service.rdf.RdfRepository; @Slf4j diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/RdfIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/RdfIndexApp.java index 515da113ba51..f27e79dcaae3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/RdfIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/RdfIndexApp.java @@ -48,7 +48,7 @@ import org.openmetadata.service.apps.bundles.rdf.distributed.RdfIndexJob; import org.openmetadata.service.exception.AppException; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipObject; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipObject; import org.openmetadata.service.jdbi3.EntityDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/distributed/DistributedRdfIndexCoordinator.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/distributed/DistributedRdfIndexCoordinator.java index 42e20359491a..5bd703868611 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/distributed/DistributedRdfIndexCoordinator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/distributed/DistributedRdfIndexCoordinator.java @@ -34,13 +34,13 @@ import org.openmetadata.service.apps.bundles.searchIndex.distributed.PartitionStatus; import org.openmetadata.service.apps.bundles.searchIndex.distributed.ServerIdentityResolver; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexJobDAO.RdfIndexJobRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexPartitionDAO.RdfAggregatedStatsRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexPartitionDAO.RdfEntityStatsRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexPartitionDAO.RdfIndexPartitionRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexPartitionDAO.RdfServerPartitionStatsRecord; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexJobDAO.RdfIndexJobRecord; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexPartitionDAO.RdfAggregatedStatsRecord; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexPartitionDAO.RdfEntityStatsRecord; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexPartitionDAO.RdfIndexPartitionRecord; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexPartitionDAO.RdfServerPartitionStatsRecord; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.RestUtil; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexCoordinator.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexCoordinator.java index 245a460b71a4..f168f1e55aed 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexCoordinator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexCoordinator.java @@ -31,16 +31,16 @@ import org.openmetadata.service.apps.bundles.searchIndex.ReindexingConfiguration; import org.openmetadata.service.apps.bundles.searchIndex.SearchIndexEntityTypes; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO.SearchIndexJobRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.AggregatedStatsRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.EntityStatsRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.SearchIndexPartitionRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.ServerStatsRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchReindexLockDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO.SearchIndexJobRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.AggregatedStatsRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.EntityStatsRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.SearchIndexPartitionRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.ServerStatsRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchReindexLockDAO; import org.openmetadata.service.util.RestUtil; /** diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryManager.java index bfd1e16024d7..86c22f3feda5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryManager.java @@ -18,7 +18,7 @@ import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchReindexLockDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchReindexLockDAO; /** * Manages recovery of distributed indexing jobs after server crashes or restarts. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/stats/JobStatsManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/stats/JobStatsManager.java index f4260437a08a..6c94d79cf623 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/stats/JobStatsManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/stats/JobStatsManager.java @@ -6,8 +6,8 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexServerStatsDAO.AggregatedServerStats; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexServerStatsDAO.EntityStats; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexServerStatsDAO.AggregatedServerStats; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexServerStatsDAO.EntityStats; /** * Manages stats trackers for all entity types within a job. Provides a simple API to get trackers diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/audit/AuditLogConsumer.java b/openmetadata-service/src/main/java/org/openmetadata/service/audit/AuditLogConsumer.java index 127f6888dbfe..b2679aedaa20 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/audit/AuditLogConsumer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/audit/AuditLogConsumer.java @@ -19,8 +19,8 @@ import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.AccessControlDAOs.ChangeEventDAO.ChangeEventRecord; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.ChangeEventDAO.ChangeEventRecord; import org.openmetadata.service.util.DIContainer; import org.quartz.DisallowConcurrentExecution; import org.quartz.Job; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/cache/BundleWarmupBatcher.java b/openmetadata-service/src/main/java/org/openmetadata/service/cache/BundleWarmupBatcher.java index 5d03db3f204f..e8e7075e0895 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/cache/BundleWarmupBatcher.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/cache/BundleWarmupBatcher.java @@ -37,8 +37,8 @@ import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipObject; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipObject; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/cache/CacheBundle.java b/openmetadata-service/src/main/java/org/openmetadata/service/cache/CacheBundle.java index 7c2c1df603d0..57b8b6a6d07b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/cache/CacheBundle.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/cache/CacheBundle.java @@ -109,7 +109,7 @@ public void run(OpenMetadataApplicationConfig configuration, Environment environ } return; } - org.openmetadata.service.jdbi3.EntityRepository.onRemoteCacheInvalidate( + org.openmetadata.service.jdbi3.EntityCacheInvalidator.onRemoteCacheInvalidate( msg.type(), msg.id(), msg.fqn()); if (msg.id() != null && cachedReadBundle != null) { cachedReadBundle.invalidate(msg.type(), msg.id()); @@ -220,7 +220,7 @@ public static void registerInvalidatable(Invalidatable layer) { /** * Fan an entity-write invalidation out to every registered {@link Invalidatable}. Today - * this is invoked from {@code EntityRepository.invalidateCacheForEntity(type, id, fqn)} + * this is invoked from {@code EntityCacheInvalidator.invalidateCacheForEntity(type, id, fqn)} * (the static helper called from {@code postCreate} and other mutation paths), from the * pub-sub handler above when a remote pod publishes a write, and from the admin * {@code POST /system/cache/invalidate} endpoint. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/cache/Invalidatable.java b/openmetadata-service/src/main/java/org/openmetadata/service/cache/Invalidatable.java index 0fece1c7ecbf..ccc8bf51ba27 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/cache/Invalidatable.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/cache/Invalidatable.java @@ -35,7 +35,7 @@ public interface Invalidatable { * have both; implementations should drop what they can. * *

Called on the local pod via {@link CacheBundle#invalidateEntity(String, UUID, String)}, - * which is wired into {@code EntityRepository.invalidateCacheForEntity} (called from + * which is wired into {@code EntityCacheInvalidator.invalidateCacheForEntity} (called from * {@code postCreate}, write-through bulk update paths, and the admin invalidate endpoint). * Note that {@code postUpdate} / {@code postDelete} / {@code restoreEntity} do NOT call * this fan-out today — they rely on the write-through cache + L1 eviction. If a new diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AccessControlDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AccessControlDAOs.java new file mode 100644 index 000000000000..600257b85564 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AccessControlDAOs.java @@ -0,0 +1,1142 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.Entity.ORGANIZATION_NAME; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.entity.Type; +import org.openmetadata.schema.entity.data.Topic; +import org.openmetadata.schema.entity.events.FailedEvent; +import org.openmetadata.schema.entity.events.FailedEventResponse; +import org.openmetadata.schema.entity.teams.Persona; +import org.openmetadata.schema.entity.teams.Role; +import org.openmetadata.schema.entity.teams.Team; +import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.ChangeEvent; +import org.openmetadata.schema.type.EventType; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.UsageDetails; +import org.openmetadata.schema.type.UsageStats; +import org.openmetadata.schema.utils.EntityInterfaceUtil; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.jdbi3.AccessControlDAOs.UsageDAO.UsageDetailsMapper; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlBatch; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.resources.events.subscription.TypedEvent; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.jdbi.BindFQN; +import org.openmetadata.service.util.jdbi.BindUUID; + +public interface AccessControlDAOs { + @CreateSqlObject + RoleDAO roleDAO(); + + @CreateSqlObject + UserDAO userDAO(); + + @CreateSqlObject + TeamDAO teamDAO(); + + @CreateSqlObject + PersonaDAO personaDAO(); + + @CreateSqlObject + UsageDAO usageDAO(); + + @CreateSqlObject + TopicDAO topicDAO(); + + @CreateSqlObject + ChangeEventDAO changeEventDAO(); + + @CreateSqlObject + TypeEntityDAO typeEntityDAO(); + + interface RoleDAO extends EntityDAO { + @Override + default String getTableName() { + return "role_entity"; + } + + @Override + default Class getEntityClass() { + return Role.class; + } + } + + interface PersonaDAO extends EntityDAO { + @Override + default String getTableName() { + return "persona_entity"; + } + + @Override + default Class getEntityClass() { + return Persona.class; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM persona_entity WHERE JSON_EXTRACT(json, '$.default') = true LIMIT 1", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT json FROM persona_entity WHERE json->>'default' = 'true' LIMIT 1", + connectionType = POSTGRES) + String findDefaultPersona(); + + @ConnectionAwareSqlQuery( + value = + "SELECT id FROM persona_entity WHERE JSON_EXTRACT(json, '$.default') = true AND id != :excludeId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT id FROM persona_entity WHERE json->>'default' = 'true' AND id != :excludeId", + connectionType = POSTGRES) + List findOtherDefaultPersonaIds(@Bind("excludeId") String excludeId); + + @ConnectionAwareSqlQuery( + value = + "SELECT id, JSON_UNQUOTE(JSON_EXTRACT(json, '$.fullyQualifiedName')) AS fqn " + + "FROM persona_entity " + + "WHERE JSON_EXTRACT(json, '$.default') = true AND id != :excludeId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT id, json->>'fullyQualifiedName' AS fqn FROM persona_entity " + + "WHERE json->>'default' = 'true' AND id != :excludeId", + connectionType = POSTGRES) + @RegisterRowMapper(EntityDAO.EntityIdFqnPairMapper.class) + List findOtherDefaultPersonaIdsWithFqn( + @Bind("excludeId") String excludeId); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE persona_entity SET json = JSON_SET(json, '$.default', false) WHERE JSON_EXTRACT(json, '$.default') = true AND id != :excludeId", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE persona_entity SET json = jsonb_set(json, '{default}', 'false') WHERE json->>'default' = 'true' AND id != :excludeId", + connectionType = POSTGRES) + void unsetOtherDefaultPersonas(@Bind("excludeId") String excludeId); + } + + interface TeamDAO extends EntityDAO { + @Override + default String getTableName() { + return "team_entity"; + } + + @Override + default Class getEntityClass() { + return Team.class; + } + + @Override + default int listCount(ListFilter filter) { + String parentTeam = filter.getQueryParam("parentTeam"); + String isJoinable = filter.getQueryParam("isJoinable"); + String condition = filter.getCondition(); + if (parentTeam != null) { + // validate parent team + Team team = findEntityByName(parentTeam, Include.ALL); + if (ORGANIZATION_NAME.equals(team.getName())) { + // All the teams without parents should come under "organization" team + condition = + String.format( + "%s AND id NOT IN ( (SELECT '%s') UNION (SELECT toId FROM entity_relationship WHERE fromId!='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d) )", + condition, team.getId(), team.getId(), Relationship.PARENT_OF.ordinal()); + } else { + condition = + String.format( + "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d)", + condition, team.getId(), Relationship.PARENT_OF.ordinal()); + } + } + String mySqlCondition = condition; + String postgresCondition = condition; + if (isJoinable != null) { + mySqlCondition = + String.format( + "%s AND JSON_EXTRACT(json, '$.isJoinable') = :isJoinable ", mySqlCondition); + postgresCondition = + String.format( + "%s AND ((json#>'{isJoinable}')::boolean) = :isJoinable ", postgresCondition); + } + + return listCount( + getTableName(), + getNameHashColumn(), + filter.getQueryParams(), + mySqlCondition, + postgresCondition); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String parentTeam = filter.getQueryParam("parentTeam"); + String isJoinable = filter.getQueryParam("isJoinable"); + String condition = filter.getCondition(); + if (parentTeam != null) { + // validate parent team + Team team = findEntityByName(parentTeam, Include.ALL); + if (ORGANIZATION_NAME.equals(team.getName())) { + // All the parentless teams should come under "organization" team + condition = + String.format( + "%s AND id NOT IN ( (SELECT '%s') UNION (SELECT toId FROM entity_relationship WHERE fromId!='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d) )", + condition, team.getId(), team.getId(), Relationship.PARENT_OF.ordinal()); + } else { + condition = + String.format( + "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d)", + condition, team.getId(), Relationship.PARENT_OF.ordinal()); + } + } + String mySqlCondition = condition; + String postgresCondition = condition; + if (isJoinable != null) { + mySqlCondition = + String.format( + "%s AND JSON_EXTRACT(json, '$.isJoinable') = :isJoinable ", mySqlCondition); + postgresCondition = + String.format( + "%s AND ((json#>'{isJoinable}')::boolean) = :isJoinable ", postgresCondition); + } + + // Quoted name is stored in fullyQualifiedName column and not in the name column + beforeName = + Optional.ofNullable(beforeName).map(FullyQualifiedName::unquoteName).orElse(null); + return listBefore( + getTableName(), + filter.getQueryParams(), + mySqlCondition, + postgresCondition, + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String parentTeam = filter.getQueryParam("parentTeam"); + String isJoinable = filter.getQueryParam("isJoinable"); + String condition = filter.getCondition(); + if (parentTeam != null) { + // validate parent team + Team team = findEntityByName(parentTeam, Include.ALL); + if (ORGANIZATION_NAME.equals(team.getName())) { + // All the parentless teams should come under "organization" team + condition = + String.format( + "%s AND id NOT IN ( (SELECT '%s') UNION (SELECT toId FROM entity_relationship WHERE fromId!='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d) )", + condition, team.getId(), team.getId(), Relationship.PARENT_OF.ordinal()); + } else { + condition = + String.format( + "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d)", + condition, team.getId(), Relationship.PARENT_OF.ordinal()); + } + } + String mySqlCondition = condition; + String postgresCondition = condition; + if (isJoinable != null) { + mySqlCondition = + String.format( + "%s AND JSON_EXTRACT(json, '$.isJoinable') = %s ", mySqlCondition, isJoinable); + postgresCondition = + String.format( + "%s AND ((json#>'{isJoinable}')::boolean) = %s ", postgresCondition, isJoinable); + } + + // Quoted name is stored in fullyQualifiedName column and not in the name column + afterName = Optional.ofNullable(afterName).map(FullyQualifiedName::unquoteName).orElse(null); + return listAfter( + getTableName(), + filter.getQueryParams(), + mySqlCondition, + postgresCondition, + limit, + afterName, + afterId); + } + + default List listTeamsUnderOrganization(UUID teamId) { + return listTeamsUnderOrganization(teamId, Relationship.PARENT_OF.ordinal()); + } + + @SqlQuery( + "SELECT te.id " + + "FROM team_entity te " + + "WHERE te.id NOT IN ((SELECT :teamId) UNION " + + "(SELECT toId FROM entity_relationship " + + "WHERE fromId != :teamId AND fromEntity = 'team' AND relation = :relation AND toEntity = 'team'))") + List listTeamsUnderOrganization( + @BindUUID("teamId") UUID teamId, @Bind("relation") int relation); + } + + interface TopicDAO extends EntityDAO { + @Override + default String getTableName() { + return "topic_entity"; + } + + @Override + default Class getEntityClass() { + return Topic.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + @RegisterRowMapper(UsageDetailsMapper.class) + interface UsageDAO { + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_usage (usageDate, id, entityType, count1, count7, count30) " + + "SELECT :date, :id, :entityType, :count1, " + + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= :date - " + + "INTERVAL 6 DAY)), " + + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= :date - " + + "INTERVAL 29 DAY))" + + "ON DUPLICATE KEY UPDATE count7 = count7 - count1 + :count1, count30 = count30 - count1 + :count1, count1 = :count1", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_usage (usageDate, id, entityType, count1, count7, count30) " + + "SELECT (:date :: date), :id, :entityType, :count1, " + + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= (:date :: date) - INTERVAL '6 days')), " + + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= (:date :: date) - INTERVAL '29 days'))" + + "ON CONFLICT (usageDate, id) DO UPDATE SET count7 = entity_usage.count7 - entity_usage.count1 + :count1," + + "count30 = entity_usage.count30 - entity_usage.count1 + :count1, count1 = :count1", + connectionType = POSTGRES) + void insertOrReplaceCount( + @Bind("date") String date, + @BindUUID("id") UUID id, + @Bind("entityType") String entityType, + @Bind("count1") int count1); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_usage (usageDate, id, entityType, count1, count7, count30) " + + "SELECT :date, :id, :entityType, :count1, " + + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= :date - " + + "INTERVAL 6 DAY)), " + + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= :date - " + + "INTERVAL 29 DAY)) " + + "ON DUPLICATE KEY UPDATE count1 = count1 + :count1, count7 = count7 + :count1, count30 = count30 + :count1", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_usage (usageDate, id, entityType, count1, count7, count30) " + + "SELECT (:date :: date), :id, :entityType, :count1, " + + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= (:date :: date) - INTERVAL '6 days')), " + + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= (:date :: date) - INTERVAL '29 days')) " + + "ON CONFLICT (usageDate, id) DO UPDATE SET count1 = entity_usage.count1 + :count1, count7 = entity_usage.count7 + :count1, count30 = entity_usage.count30 + :count1", + connectionType = POSTGRES) + void insertOrUpdateCount( + @Bind("date") String date, + @BindUUID("id") UUID id, + @Bind("entityType") String entityType, + @Bind("count1") int count1); + + @ConnectionAwareSqlQuery( + value = + "SELECT id, usageDate, entityType, count1, count7, count30, " + + "percentile1, percentile7, percentile30 FROM entity_usage " + + "WHERE id = :id AND usageDate >= :date - INTERVAL :days DAY AND usageDate <= :date ORDER BY usageDate DESC", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT id, usageDate, entityType, count1, count7, count30, " + + "percentile1, percentile7, percentile30 FROM entity_usage " + + "WHERE id = :id AND usageDate >= (:date :: date) - make_interval(days => :days) AND usageDate <= (:date :: date) ORDER BY usageDate DESC", + connectionType = POSTGRES) + List getUsageById( + @BindUUID("id") UUID id, @Bind("date") String date, @Bind("days") int days); + + /** Get latest usage record */ + @SqlQuery( + "SELECT id, usageDate, entityType, count1, count7, count30, " + + "percentile1, percentile7, percentile30 FROM entity_usage " + + "WHERE usageDate IN (SELECT MAX(usageDate) FROM entity_usage WHERE id = :id) AND id = :id") + UsageDetails getLatestUsage(@Bind("id") String id); + + /** Get latest usage records for multiple entities in one query */ + @RegisterRowMapper(UsageDetailsWithIdMapper.class) + @SqlQuery( + "SELECT u1.id, u1.usageDate, u1.entityType, u1.count1, u1.count7, u1.count30, " + + "u1.percentile1, u1.percentile7, u1.percentile30 FROM entity_usage u1 " + + "INNER JOIN (SELECT id, MAX(usageDate) as maxDate FROM entity_usage WHERE id IN () GROUP BY id) u2 " + + "ON u1.id = u2.id AND u1.usageDate = u2.maxDate") + List getLatestUsageBatch(@BindList("ids") List ids); + + @SqlUpdate("DELETE FROM entity_usage WHERE id = :id") + void delete(@BindUUID("id") UUID id); + + /** + * TODO: Not sure I get what the next comment means, but tests now use mysql 8 so maybe tests can be improved here + * Note not using in following percentile computation PERCENT_RANK function as unit tests use mysql5.7, and it does + * not have window function + */ + @ConnectionAwareSqlUpdate( + value = + "UPDATE entity_usage u JOIN ( " + + "SELECT u1.id, " + + "(SELECT COUNT(*) FROM entity_usage as u2 WHERE u2.count1 < u1.count1 AND u2.entityType = :entityType " + + "AND u2.usageDate = :date) as p1, " + + "(SELECT COUNT(*) FROM entity_usage as u3 WHERE u3.count7 < u1.count7 AND u3.entityType = :entityType " + + "AND u3.usageDate = :date) as p7, " + + "(SELECT COUNT(*) FROM entity_usage as u4 WHERE u4.count30 < u1.count30 AND u4.entityType = :entityType " + + "AND u4.usageDate = :date) as p30, " + + "(SELECT COUNT(*) FROM entity_usage WHERE entityType = :entityType AND usageDate = :date) as total " + + "FROM entity_usage u1 WHERE u1.entityType = :entityType AND u1.usageDate = :date" + + ") vals ON u.id = vals.id AND usageDate = :date " + + "SET u.percentile1 = ROUND(100 * p1/total, 2), u.percentile7 = ROUND(p7 * 100/total, 2), u.percentile30 =" + + " ROUND(p30*100/total, 2)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE entity_usage u " + + "SET percentile1 = ROUND(100 * p1 / total, 2), percentile7 = ROUND(p7 * 100 / total, 2), percentile30 = ROUND(p30 * 100 / total, 2) " + + "FROM (" + + " SELECT u1.id, " + + " (SELECT COUNT(*) FROM entity_usage as u2 WHERE u2.count1 < u1.count1 AND u2.entityType = :entityType AND u2.usageDate = (:date :: date)) as p1, " + + " (SELECT COUNT(*) FROM entity_usage as u3 WHERE u3.count7 < u1.count7 AND u3.entityType = :entityType AND u3.usageDate = (:date :: date)) as p7, " + + " (SELECT COUNT(*) FROM entity_usage as u4 WHERE u4.count30 < u1.count30 AND u4.entityType = :entityType AND u4.usageDate = (:date :: date)) as p30, " + + " (SELECT COUNT(*) FROM entity_usage WHERE entityType = :entityType AND usageDate = (:date :: date)" + + " ) as total FROM entity_usage u1 " + + " WHERE u1.entityType = :entityType AND u1.usageDate = (:date :: date)" + + ") vals " + + "WHERE u.id = vals.id AND usageDate = (:date :: date);", + connectionType = POSTGRES) + void computePercentile(@Bind("entityType") String entityType, @Bind("date") String date); + + class UsageDetailsMapper implements RowMapper { + @Override + public UsageDetails map(ResultSet r, StatementContext ctx) throws SQLException { + UsageStats dailyStats = + new UsageStats() + .withCount(r.getInt("count1")) + .withPercentileRank(r.getDouble("percentile1")); + UsageStats weeklyStats = + new UsageStats() + .withCount(r.getInt("count7")) + .withPercentileRank(r.getDouble("percentile7")); + UsageStats monthlyStats = + new UsageStats() + .withCount(r.getInt("count30")) + .withPercentileRank(r.getDouble("percentile30")); + java.sql.Date usageDate = r.getDate("usageDate"); + return new UsageDetails() + .withDate(usageDate != null ? usageDate.toString() : null) + .withDailyStats(dailyStats) + .withWeeklyStats(weeklyStats) + .withMonthlyStats(monthlyStats); + } + } + + /** Usage details with entity ID for batch operations */ + class UsageDetailsWithId { + private final String entityId; + private final UsageDetails usageDetails; + + public UsageDetailsWithId(String entityId, UsageDetails usageDetails) { + this.entityId = entityId; + this.usageDetails = usageDetails; + } + + public String getEntityId() { + return entityId; + } + + public UsageDetails getUsageDetails() { + return usageDetails; + } + } + + class UsageDetailsWithIdMapper implements RowMapper { + @Override + public UsageDetailsWithId map(ResultSet r, StatementContext ctx) throws SQLException { + String entityId = r.getString("id"); + UsageStats dailyStats = + new UsageStats() + .withCount(r.getInt("count1")) + .withPercentileRank(r.getDouble("percentile1")); + UsageStats weeklyStats = + new UsageStats() + .withCount(r.getInt("count7")) + .withPercentileRank(r.getDouble("percentile7")); + UsageStats monthlyStats = + new UsageStats() + .withCount(r.getInt("count30")) + .withPercentileRank(r.getDouble("percentile30")); + java.sql.Date usageDate = r.getDate("usageDate"); + UsageDetails usageDetails = + new UsageDetails() + .withDate(usageDate != null ? usageDate.toString() : null) + .withDailyStats(dailyStats) + .withWeeklyStats(weeklyStats) + .withMonthlyStats(monthlyStats); + return new UsageDetailsWithId(entityId, usageDetails); + } + } + } + + interface UserDAO extends EntityDAO { + @Override + default String getTableName() { + return "user_entity"; + } + + @Override + default Class getEntityClass() { + return User.class; + } + + @Override + default int listCount(ListFilter filter) { + String team = EntityInterfaceUtil.quoteName(filter.getQueryParam("team")); + String isBotStr = filter.getQueryParam("isBot"); + String isAdminStr = filter.getQueryParam("isAdmin"); + String lastLoginTimeGreaterThan = filter.getQueryParam("lastLoginTimeGreaterThan"); + String lastActivityTimeGreaterThan = filter.getQueryParam("lastActivityTimeGreaterThan"); + String mySqlCondition = filter.getCondition("ue"); + String postgresCondition = filter.getCondition("ue"); + if (isAdminStr != null) { + boolean isAdmin = Boolean.parseBoolean(isAdminStr); + if (isAdmin) { + mySqlCondition = + String.format("%s AND JSON_EXTRACT(ue.json, '$.isAdmin') = TRUE ", mySqlCondition); + postgresCondition = + String.format("%s AND ((ue.json#>'{isAdmin}')::boolean) = TRUE ", postgresCondition); + } else { + mySqlCondition = + String.format( + "%s AND (JSON_EXTRACT(ue.json, '$.isAdmin') IS NULL OR JSON_EXTRACT(ue.json, '$.isAdmin') = FALSE ) ", + mySqlCondition); + postgresCondition = + String.format( + "%s AND (ue.json#>'{isAdmin}' IS NULL OR ((ue.json#>'{isAdmin}')::boolean) = FALSE ) ", + postgresCondition); + } + } + if (isBotStr != null) { + boolean isBot = Boolean.parseBoolean(isBotStr); + if (isBot) { + mySqlCondition = String.format("%s AND ue.isBot = TRUE ", mySqlCondition); + postgresCondition = String.format("%s AND ue.isBot = TRUE ", postgresCondition); + } else { + mySqlCondition = String.format("%s AND ue.isBot = FALSE ", mySqlCondition); + postgresCondition = String.format("%s AND ue.isBot = FALSE ", postgresCondition); + } + } + if (lastLoginTimeGreaterThan != null) { + mySqlCondition = + String.format( + "%s AND ue.lastLoginTime > %s ", mySqlCondition, lastLoginTimeGreaterThan); + postgresCondition = + String.format( + "%s AND ue.lastLoginTime > %s ", postgresCondition, lastLoginTimeGreaterThan); + } + if (lastActivityTimeGreaterThan != null) { + mySqlCondition = + String.format( + "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", + mySqlCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); + postgresCondition = + String.format( + "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", + postgresCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); + } + if (team == null + && isAdminStr == null + && isBotStr == null + && lastLoginTimeGreaterThan == null + && lastActivityTimeGreaterThan == null) { + return EntityDAO.super.listCount(filter); + } + return listCount( + getTableName(), mySqlCondition, postgresCondition, team, Relationship.HAS.ordinal()); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String team = EntityInterfaceUtil.quoteName(filter.getQueryParam("team")); + String isBotStr = filter.getQueryParam("isBot"); + String isAdminStr = filter.getQueryParam("isAdmin"); + String lastLoginTimeGreaterThan = filter.getQueryParam("lastLoginTimeGreaterThan"); + String lastActivityTimeGreaterThan = filter.getQueryParam("lastActivityTimeGreaterThan"); + String mySqlCondition = filter.getCondition("ue"); + String postgresCondition = filter.getCondition("ue"); + if (isAdminStr != null) { + boolean isAdmin = Boolean.parseBoolean(isAdminStr); + if (isAdmin) { + mySqlCondition = + String.format("%s AND JSON_EXTRACT(ue.json, '$.isAdmin') = TRUE ", mySqlCondition); + postgresCondition = + String.format("%s AND ((ue.json#>'{isAdmin}')::boolean) = TRUE ", postgresCondition); + } else { + mySqlCondition = + String.format( + "%s AND (JSON_EXTRACT(ue.json, '$.isAdmin') IS NULL OR JSON_EXTRACT(ue.json, '$.isAdmin') = FALSE ) ", + mySqlCondition); + postgresCondition = + String.format( + "%s AND (ue.json#>'{isAdmin}' IS NULL OR ((ue.json#>'{isAdmin}')::boolean) = FALSE ) ", + postgresCondition); + } + } + if (isBotStr != null) { + boolean isBot = Boolean.parseBoolean(isBotStr); + if (isBot) { + mySqlCondition = String.format("%s AND ue.isBot = TRUE ", mySqlCondition); + postgresCondition = String.format("%s AND ue.isBot = TRUE ", postgresCondition); + } else { + mySqlCondition = String.format("%s AND ue.isBot = FALSE ", mySqlCondition); + postgresCondition = String.format("%s AND ue.isBot = FALSE ", postgresCondition); + } + } + if (lastLoginTimeGreaterThan != null) { + mySqlCondition = + String.format( + "%s AND ue.lastLoginTime > %s ", mySqlCondition, lastLoginTimeGreaterThan); + postgresCondition = + String.format( + "%s AND ue.lastLoginTime > %s ", postgresCondition, lastLoginTimeGreaterThan); + } + if (lastActivityTimeGreaterThan != null) { + mySqlCondition = + String.format( + "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", + mySqlCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); + postgresCondition = + String.format( + "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", + postgresCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); + } + if (team == null + && isAdminStr == null + && isBotStr == null + && lastLoginTimeGreaterThan == null + && lastActivityTimeGreaterThan == null) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + return listBefore( + getTableName(), + mySqlCondition, + postgresCondition, + team, + limit, + beforeName, + beforeId, + Relationship.HAS.ordinal()); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String team = EntityInterfaceUtil.quoteName(filter.getQueryParam("team")); + String isBotStr = filter.getQueryParam("isBot"); + String isAdminStr = filter.getQueryParam("isAdmin"); + String lastLoginTimeGreaterThan = filter.getQueryParam("lastLoginTimeGreaterThan"); + String lastActivityTimeGreaterThan = filter.getQueryParam("lastActivityTimeGreaterThan"); + String mySqlCondition = filter.getCondition("ue"); + String postgresCondition = filter.getCondition("ue"); + if (isAdminStr != null) { + boolean isAdmin = Boolean.parseBoolean(isAdminStr); + if (isAdmin) { + mySqlCondition = + String.format("%s AND JSON_EXTRACT(ue.json, '$.isAdmin') = TRUE ", mySqlCondition); + postgresCondition = + String.format("%s AND ((ue.json#>'{isAdmin}')::boolean) = TRUE ", postgresCondition); + } else { + mySqlCondition = + String.format( + "%s AND (JSON_EXTRACT(ue.json, '$.isAdmin') IS NULL OR JSON_EXTRACT(ue.json, '$.isAdmin') = FALSE ) ", + mySqlCondition); + postgresCondition = + String.format( + "%s AND (ue.json#>'{isAdmin}' IS NULL OR ((ue.json#>'{isAdmin}')::boolean) = FALSE ) ", + postgresCondition); + } + } + if (isBotStr != null) { + boolean isBot = Boolean.parseBoolean(isBotStr); + if (isBot) { + mySqlCondition = String.format("%s AND ue.isBot = TRUE ", mySqlCondition); + postgresCondition = String.format("%s AND ue.isBot = TRUE ", postgresCondition); + } else { + mySqlCondition = String.format("%s AND ue.isBot = FALSE ", mySqlCondition); + postgresCondition = String.format("%s AND ue.isBot = FALSE ", postgresCondition); + } + } + if (lastLoginTimeGreaterThan != null) { + mySqlCondition = + String.format( + "%s AND ue.lastLoginTime > %s ", mySqlCondition, lastLoginTimeGreaterThan); + postgresCondition = + String.format( + "%s AND ue.lastLoginTime > %s ", postgresCondition, lastLoginTimeGreaterThan); + } + if (lastActivityTimeGreaterThan != null) { + mySqlCondition = + String.format( + "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", + mySqlCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); + postgresCondition = + String.format( + "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", + postgresCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); + } + if (team == null + && isAdminStr == null + && isBotStr == null + && lastLoginTimeGreaterThan == null + && lastActivityTimeGreaterThan == null) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + return listAfter( + getTableName(), + mySqlCondition, + postgresCondition, + team, + limit, + afterName, + afterId, + Relationship.HAS.ordinal()); + } + + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM (" + + "SELECT ue.id " + + "FROM user_entity ue " + + "LEFT JOIN entity_relationship er on ue.id = er.toId " + + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " + + " " + + " AND (:team IS NULL OR te.nameHash = :team) " + + "GROUP BY ue.id) subquery", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM (" + + "SELECT ue.id " + + "FROM user_entity ue " + + "LEFT JOIN entity_relationship er on ue.id = er.toId " + + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " + + " " + + " AND (:team IS NULL OR te.nameHash = :team) " + + "GROUP BY ue.id) subquery", + connectionType = POSTGRES) + int listCount( + @Define("table") String table, + @Define("mysqlCond") String mysqlCond, + @Define("postgresCond") String postgresCond, + @BindFQN("team") String team, + @Bind("relation") int relation); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT ue.name, ue.id, ue.json " + + "FROM user_entity ue " + + "LEFT JOIN entity_relationship er on ue.id = er.toId " + + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " + + " " + + "AND (:team IS NULL OR te.nameHash = :team) " + + "AND (ue.name < :beforeName OR (ue.name = :beforeName AND ue.id < :beforeId)) " + + "GROUP BY ue.name, ue.id, ue.json " + + "ORDER BY ue.name DESC,ue.id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT ue.name, ue.id, ue.json " + + "FROM user_entity ue " + + "LEFT JOIN entity_relationship er on ue.id = er.toId " + + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " + + " " + + "AND (:team IS NULL OR te.nameHash = :team) " + + "AND (ue.name < :beforeName OR (ue.name = :beforeName AND ue.id < :beforeId)) " + + "GROUP BY ue.name, ue.id, ue.json " + + "ORDER BY ue.name DESC,ue.id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id", + connectionType = POSTGRES) + List listBefore( + @Define("table") String table, + @Define("mysqlCond") String mysqlCond, + @Define("postgresCond") String postgresCond, + @BindFQN("team") String team, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId, + @Bind("relation") int relation); + + @ConnectionAwareSqlQuery( + value = + "SELECT ue.json " + + "FROM user_entity ue " + + "LEFT JOIN entity_relationship er on ue.id = er.toId " + + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " + + " " + + "AND (:team IS NULL OR te.nameHash = :team) " + + "AND (ue.name > :afterName OR (ue.name = :afterName AND ue.id > :afterId)) " + + "GROUP BY ue.name, ue.id, ue.json " + + "ORDER BY ue.name,ue.id " + + "LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT ue.json " + + "FROM user_entity ue " + + "LEFT JOIN entity_relationship er on ue.id = er.toId " + + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " + + " " + + "AND (:team IS NULL OR te.nameHash = :team) " + + "AND (ue.name > :afterName OR (ue.name = :afterName AND ue.id > :afterId)) " + + "GROUP BY ue.name,ue.id, ue.json " + + "ORDER BY ue.name,ue.id " + + "LIMIT :limit", + connectionType = POSTGRES) + List listAfter( + @Define("table") String table, + @Define("mysqlCond") String mysqlCond, + @Define("postgresCond") String postgresCond, + @BindFQN("team") String team, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId, + @Bind("relation") int relation); + + @SqlQuery("SELECT COUNT(*) FROM user_entity WHERE LOWER(email) = LOWER(:email)") + int checkEmailExists(@Bind("email") String email); + + @SqlQuery("SELECT COUNT(*) FROM user_entity WHERE LOWER(name) = LOWER(:name)") + int checkUserNameExists(@Bind("name") String name); + + @SqlQuery( + "SELECT json FROM user_entity WHERE LOWER(name) = LOWER(:name) AND LOWER(email) = LOWER(:email)") + String findUserByNameAndEmail(@Bind("name") String name, @Bind("email") String email); + + @SqlQuery("SELECT json FROM user_entity WHERE LOWER(email) = LOWER(:email)") + String findUserByEmail(@Bind("email") String email); + + @Override + default User findEntityByName(String fqn, Include include) { + return EntityDAO.super.findEntityByName(fqn.toLowerCase(), include); + } + + @ConnectionAwareSqlUpdate( + value = + "UPDATE user_entity SET json = JSON_SET(json, '$.lastActivityTime', :lastActivityTime) WHERE nameHash = :nameHash AND deleted = false", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE user_entity SET json = jsonb_set(json, '{lastActivityTime}', to_jsonb(:lastActivityTime::bigint)) WHERE nameHash = :nameHash AND deleted = false", + connectionType = POSTGRES) + void updateLastActivityTime( + @BindFQN("nameHash") String nameHash, @Bind("lastActivityTime") long lastActivityTime); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE user_entity SET json = JSON_SET(json, '$.lastActivityTime', " + + "CASE nameHash " + + " " + + "END) " + + "WHERE nameHash IN () AND deleted = false", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE user_entity SET json = jsonb_set(json, '{lastActivityTime}', " + + "CASE nameHash " + + " " + + "END::text::jsonb) " + + "WHERE nameHash IN () AND deleted = false", + connectionType = POSTGRES) + void updateLastActivityTimeBulk( + @Define("caseStatements") String caseStatements, + @BindList("nameHashes") List nameHashes); + + @SqlQuery( + "SELECT lastActivityTime FROM user_entity " + + "WHERE isBot = FALSE AND lastActivityTime IS NOT NULL AND deleted = FALSE " + + "ORDER BY lastActivityTime DESC LIMIT 1") + Long getMaxLastActivityTime(); + + @SqlQuery( + "SELECT COUNT(DISTINCT id) FROM user_entity " + + "WHERE isBot = false " + + "AND deleted = false " + + "AND lastActivityTime >= :since") + int countDailyActiveUsers(@Bind("since") long since); + } + + interface ChangeEventDAO { + @SqlQuery( + "SELECT json FROM change_event ce where ce.offset > :offset ORDER BY ce.eventTime DESC LIMIT :limit OFFSET :paginationOffset") + List listUnprocessedEvents( + @Bind("offset") long offset, + @Bind("limit") int limit, + @Bind("paginationOffset") int paginationOffset); + + @SqlQuery( + "SELECT json, source FROM consumers_dlq WHERE id = :id ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") + @RegisterRowMapper(FailedEventResponseMapper.class) + List listFailedEventsById( + @Bind("id") String id, + @Bind("limit") int limit, + @Bind("paginationOffset") int paginationOffset); + + @SqlQuery("SELECT COUNT(*) FROM consumers_dlq WHERE id = :id") + long countFailedEvents(@Bind("id") String id); + + @SqlQuery( + "SELECT json, source FROM consumers_dlq WHERE id = :id AND source = :source ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") + @RegisterRowMapper(FailedEventResponseMapper.class) + List listFailedEventsByIdAndSource( + @Bind("id") String id, + @Bind("source") String source, + @Bind("limit") int limit, + @Bind("paginationOffset") int paginationOffset); + + @SqlQuery( + "SELECT json, source FROM consumers_dlq ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") + @RegisterRowMapper(FailedEventResponseMapper.class) + List listAllFailedEvents( + @Bind("limit") int limit, @Bind("paginationOffset") int paginationOffset); + + @SqlQuery( + "SELECT json, source FROM consumers_dlq WHERE source = :source ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") + @RegisterRowMapper(FailedEventResponseMapper.class) + List listAllFailedEventsBySource( + @Bind("source") String source, + @Bind("limit") int limit, + @Bind("paginationOffset") int paginationOffset); + + @ConnectionAwareSqlQuery( + value = + "SELECT json, status, timestamp " + + "FROM ( " + + " SELECT json, 'FAILED' AS status, timestamp " + + " FROM consumers_dlq WHERE id = :id " + + " UNION ALL " + + " SELECT json, 'SUCCESSFUL' AS status, timestamp " + + " FROM successful_sent_change_events WHERE event_subscription_id = :id " + + ") AS combined_events " + + "ORDER BY timestamp DESC " + + "LIMIT :limit OFFSET :paginationOffset", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json, status, timestamp " + + "FROM ( " + + " SELECT json, 'failed' AS status, timestamp " + + " FROM consumers_dlq WHERE id = :id " + + " UNION ALL " + + " SELECT json, 'successful' AS status, timestamp " + + " FROM successful_sent_change_events WHERE event_subscription_id = :id " + + ") AS combined_events " + + "ORDER BY timestamp DESC " + + "LIMIT :limit OFFSET :paginationOffset", + connectionType = POSTGRES) + @RegisterRowMapper(EventResponseMapper.class) + List listAllEventsWithStatuses( + @Bind("id") String id, + @Bind("limit") int limit, + @Bind("paginationOffset") long paginationOffset); + + @SqlQuery("SELECT json FROM change_event ce where ce.offset > :offset") + List listUnprocessedEvents(@Bind("offset") long offset); + + @SqlQuery( + "SELECT CASE WHEN EXISTS (SELECT 1 FROM event_subscription_entity WHERE id = :id) THEN 1 ELSE 0 END AS record_exists") + int recordExists(@Bind("id") String id); + + @ConnectionAwareSqlUpdate( + value = "INSERT INTO change_event (json) VALUES (:json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO change_event (json) VALUES (:json :: jsonb)", + connectionType = POSTGRES) + void insert(@Bind("json") String json); + + @Transaction + @ConnectionAwareSqlBatch( + value = "INSERT INTO change_event (json) VALUES (:json)", + connectionType = MYSQL) + @ConnectionAwareSqlBatch( + value = "INSERT INTO change_event (json) VALUES (CAST(:json AS jsonb))", + connectionType = POSTGRES) + void insertBatchRows(@Bind("json") List jsons); + + default void insertBatch(List jsons) { + if (nullOrEmpty(jsons)) { + return; + } + insertBatchRows(jsons); + } + + @SqlUpdate("DELETE FROM change_event WHERE entityType = :entityType") + void deleteAll(@Bind("entityType") String entityType); + + default List list(EventType eventType, List entityTypes, long timestamp) { + if (nullOrEmpty(entityTypes)) { + return Collections.emptyList(); + } + if (entityTypes.get(0).equals("*")) { + return listWithoutEntityFilter(eventType.value(), timestamp); + } + return listWithEntityFilter(eventType.value(), entityTypes, timestamp); + } + + @SqlQuery( + "SELECT json FROM change_event WHERE " + + "eventType = :eventType AND (entityType IN ()) AND eventTime >= :timestamp " + + "ORDER BY eventTime ASC") + List listWithEntityFilter( + @Bind("eventType") String eventType, + @BindList("entityTypes") List entityTypes, + @Bind("timestamp") long timestamp); + + @SqlQuery( + "SELECT json FROM change_event WHERE " + + "eventType = :eventType AND eventTime >= :timestamp " + + "ORDER BY eventTime ASC") + List listWithoutEntityFilter( + @Bind("eventType") String eventType, @Bind("timestamp") long timestamp); + + @SqlQuery( + "SELECT json FROM change_event ce WHERE ce.offset > :offset ORDER BY ce.offset ASC LIMIT :limit") + List list(@Bind("limit") long limit, @Bind("offset") long offset); + + @ConnectionAwareSqlQuery(value = "SELECT MAX(offset) FROM change_event", connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT MAX(\"offset\") FROM change_event", + connectionType = POSTGRES) + long getLatestOffset(); + + @SqlQuery("SELECT count(*) FROM change_event") + long listCount(); + + /** Record holding change event offset and JSON for cursor-based pagination. */ + record ChangeEventRecord(long offset, String json) {} + + /** Returns change events with their offset values for accurate cursor tracking. */ + @ConnectionAwareSqlQuery( + value = + "SELECT `offset`, json FROM change_event WHERE `offset` > :afterOffset ORDER BY `offset` ASC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT \"offset\", json FROM change_event WHERE \"offset\" > :afterOffset ORDER BY \"offset\" ASC LIMIT :limit", + connectionType = POSTGRES) + @RegisterRowMapper(ChangeEventRecordMapper.class) + List listWithOffset( + @Bind("limit") int limit, @Bind("afterOffset") long afterOffset); + } + + class ChangeEventRecordMapper implements RowMapper { + @Override + public ChangeEventDAO.ChangeEventRecord map(ResultSet rs, StatementContext ctx) + throws SQLException { + return new ChangeEventDAO.ChangeEventRecord(rs.getLong("offset"), rs.getString("json")); + } + } + + class FailedEventResponseMapper implements RowMapper { + @Override + public FailedEventResponse map(ResultSet rs, StatementContext ctx) throws SQLException { + FailedEventResponse response = new FailedEventResponse(); + FailedEvent failedEvent = JsonUtils.readValue(rs.getString("json"), FailedEvent.class); + response.setFailingSubscriptionId(failedEvent.getFailingSubscriptionId()); + response.setChangeEvent(failedEvent.getChangeEvent()); + response.setReason(failedEvent.getReason()); + response.setSource(rs.getString("source")); + response.setTimestamp(failedEvent.getTimestamp()); + return response; + } + } + + class EventResponseMapper implements RowMapper { + @Override + public TypedEvent map(ResultSet rs, StatementContext ctx) throws SQLException { + TypedEvent response = new TypedEvent(); + String status = rs.getString("status").toLowerCase(); + + if (TypedEvent.Status.FAILED.value().equalsIgnoreCase(status)) { + FailedEvent failedEvent = JsonUtils.readValue(rs.getString("json"), FailedEvent.class); + response.setData(List.of(failedEvent)); + response.setStatus(TypedEvent.Status.FAILED); + } else { + ChangeEvent changeEvent = JsonUtils.readValue(rs.getString("json"), ChangeEvent.class); + response.setData(List.of(changeEvent)); + response.setStatus(TypedEvent.Status.fromValue(status)); + } + + long timestampMillis = rs.getLong("timestamp"); + response.setTimestamp((double) timestampMillis); + return response; + } + } + + interface TypeEntityDAO extends EntityDAO { + @Override + default String getTableName() { + return "type_entity"; + } + + @Override + default Class getEntityClass() { + return Type.class; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ActivityAuditDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ActivityAuditDAOs.java new file mode 100644 index 000000000000..2bf9266fc594 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ActivityAuditDAOs.java @@ -0,0 +1,509 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.util.List; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindBean; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.customizer.BindMethods; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.service.audit.AuditLogRecord; +import org.openmetadata.service.audit.AuditLogRecordMapper; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlBatch; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; + +public interface ActivityAuditDAOs { + @CreateSqlObject + ActivityStreamDAO activityStreamDAO(); + + @CreateSqlObject + ActivityStreamConfigDAO activityStreamConfigDAO(); + + @CreateSqlObject + AuditLogDAO auditLogDAO(); + + interface ActivityStreamDAO { + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO activity_stream(id, eventType, entityType, entityId, entityFqnHash, " + + "about, aboutFqnHash, actorId, actorName, timestamp, summary, fieldName, oldValue, newValue, domains, json) " + + "VALUES (:id, :eventType, :entityType, :entityId, :entityFqnHash, " + + ":about, :aboutFqnHash, :actorId, :actorName, :timestamp, :summary, :fieldName, :oldValue, :newValue, :domains, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO activity_stream(id, eventtype, entitytype, entityid, entityfqnhash, " + + "about, aboutfqnhash, actorid, actorname, timestamp, summary, fieldname, oldvalue, newvalue, domains, json) " + + "VALUES (:id, :eventType, :entityType, :entityId, :entityFqnHash, " + + ":about, :aboutFqnHash, :actorId, :actorName, :timestamp, :summary, :fieldName, :oldValue, :newValue, :domains::jsonb, :json::jsonb)", + connectionType = POSTGRES) + void insert( + @Bind("id") String id, + @Bind("eventType") String eventType, + @Bind("entityType") String entityType, + @Bind("entityId") String entityId, + @Bind("entityFqnHash") String entityFqnHash, + @Bind("about") String about, + @Bind("aboutFqnHash") String aboutFqnHash, + @Bind("actorId") String actorId, + @Bind("actorName") String actorName, + @Bind("timestamp") long timestamp, + @Bind("summary") String summary, + @Bind("fieldName") String fieldName, + @Bind("oldValue") String oldValue, + @Bind("newValue") String newValue, + @Bind("domains") String domains, + @Bind("json") String json); + + // Batch insert for activity events - one round-trip per change event instead of one per row + @Transaction + @ConnectionAwareSqlBatch( + value = + "INSERT INTO activity_stream(id, eventType, entityType, entityId, entityFqnHash, " + + "about, aboutFqnHash, actorId, actorName, timestamp, summary, fieldName, oldValue, newValue, domains, json) " + + "VALUES (:id, :eventType, :entityType, :entityId, :entityFqnHash, " + + ":about, :aboutFqnHash, :actorId, :actorName, :timestamp, :summary, :fieldName, :oldValue, :newValue, :domains, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlBatch( + value = + "INSERT INTO activity_stream(id, eventtype, entitytype, entityid, entityfqnhash, " + + "about, aboutfqnhash, actorid, actorname, timestamp, summary, fieldname, oldvalue, newvalue, domains, json) " + + "VALUES (:id, :eventType, :entityType, :entityId, :entityFqnHash, " + + ":about, :aboutFqnHash, :actorId, :actorName, :timestamp, :summary, :fieldName, :oldValue, :newValue, :domains::jsonb, :json::jsonb)", + connectionType = POSTGRES) + void insertBatch(@BindMethods List rows); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE timestamp >= :after " + + "ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE timestamp >= :after " + + "ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List list(@Bind("after") long after, @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entityType = :entityType AND entityId = :entityId " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entitytype = :entityType AND entityid = :entityId " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByEntity( + @Bind("entityType") String entityType, + @Bind("entityId") String entityId, + @Bind("after") long after, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entityType = :entityType " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entitytype = :entityType " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByEntityType( + @Bind("entityType") String entityType, @Bind("after") long after, @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE actorId = :actorId " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE actorid = :actorId " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByActor( + @Bind("actorId") String actorId, @Bind("after") long after, @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE actorId = :actorId " + + "AND JSON_OVERLAPS(domains, :domainJson) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE actorid = :actorId " + + "AND EXISTS (" + + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " + + "WHERE domain_id IN ()) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByActorAndDomains( + @Bind("actorId") String actorId, + @Bind("domainJson") String domainJson, + @BindList("domainIds") List domainIds, + @Bind("after") long after, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE JSON_OVERLAPS(domains, :domainJson) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE EXISTS (" + + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " + + "WHERE domain_id IN ()) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByDomains( + @Bind("domainJson") String domainJson, + @BindList("domainIds") List domainIds, + @Bind("after") long after, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entityType = :entityType AND entityId = :entityId " + + "AND JSON_OVERLAPS(domains, :domainJson) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entitytype = :entityType AND entityid = :entityId " + + "AND EXISTS (" + + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " + + "WHERE domain_id IN ()) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByEntityAndDomains( + @Bind("entityType") String entityType, + @Bind("entityId") String entityId, + @Bind("domainJson") String domainJson, + @BindList("domainIds") List domainIds, + @Bind("after") long after, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entityType = :entityType " + + "AND JSON_OVERLAPS(domains, :domainJson) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entitytype = :entityType " + + "AND EXISTS (" + + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " + + "WHERE domain_id IN ()) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByEntityTypeAndDomains( + @Bind("entityType") String entityType, + @Bind("domainJson") String domainJson, + @BindList("domainIds") List domainIds, + @Bind("after") long after, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM activity_stream WHERE timestamp >= :after", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM activity_stream WHERE timestamp >= :after", + connectionType = POSTGRES) + int count(@Bind("after") long after); + + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM activity_stream WHERE JSON_OVERLAPS(domains, :domainJson) " + + "AND timestamp >= :after", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM activity_stream WHERE EXISTS (" + + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " + + "WHERE domain_id IN ()) " + + "AND timestamp >= :after", + connectionType = POSTGRES) + int countByDomains( + @Bind("domainJson") String domainJson, + @BindList("domainIds") List domainIds, + @Bind("after") long after); + + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM activity_stream WHERE entityType = :entityType AND entityId = :entityId " + + "AND timestamp >= :after", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM activity_stream WHERE entitytype = :entityType AND entityid = :entityId " + + "AND timestamp >= :after", + connectionType = POSTGRES) + int countByEntity( + @Bind("entityType") String entityType, + @Bind("entityId") String entityId, + @Bind("after") long after); + + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM activity_stream WHERE entityType = :entityType AND entityId = :entityId " + + "AND JSON_OVERLAPS(domains, :domainJson) " + + "AND timestamp >= :after", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM activity_stream WHERE entitytype = :entityType AND entityid = :entityId " + + "AND EXISTS (" + + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " + + "WHERE domain_id IN ()) " + + "AND timestamp >= :after", + connectionType = POSTGRES) + int countByEntityAndDomains( + @Bind("entityType") String entityType, + @Bind("entityId") String entityId, + @Bind("domainJson") String domainJson, + @BindList("domainIds") List domainIds, + @Bind("after") long after); + + @SqlUpdate("DELETE FROM activity_stream WHERE timestamp < :cutoff") + int deleteOlderThan(@Bind("cutoff") long cutoffTimestamp); + + @SqlQuery("SELECT json FROM activity_stream WHERE id = :id") + String findById(@Bind("id") String id); + + @ConnectionAwareSqlUpdate( + value = "UPDATE activity_stream SET json = :json WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE activity_stream SET json = :json::jsonb WHERE id = :id", + connectionType = POSTGRES) + void updateJson(@Bind("id") String id, @Bind("json") String json); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entityId IN (" + + "SELECT toId FROM entity_relationship WHERE relation = 8 " + + "AND ((fromEntity = 'user' AND fromId = :userId) " + + "OR (fromEntity = 'team' AND fromId IN ()))) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entityid IN (" + + "SELECT toid FROM entity_relationship WHERE relation = 8 " + + "AND ((fromentity = 'user' AND fromid = :userId) " + + "OR (fromentity = 'team' AND fromid IN ()))) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByOwners( + @Bind("userId") String userId, + @BindList("teamIds") List teamIds, + @Bind("after") long after, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entityId IN (" + + "SELECT toId FROM entity_relationship WHERE relation = 8 " + + "AND ((fromEntity = 'user' AND fromId = :userId) " + + "OR (fromEntity = 'team' AND fromId IN ()))) " + + "AND JSON_OVERLAPS(domains, :domainJson) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE entityid IN (" + + "SELECT toid FROM entity_relationship WHERE relation = 8 " + + "AND ((fromentity = 'user' AND fromid = :userId) " + + "OR (fromentity = 'team' AND fromid IN ()))) " + + "AND EXISTS (" + + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " + + "WHERE domain_id IN ()) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByOwnersAndDomains( + @Bind("userId") String userId, + @BindList("teamIds") List teamIds, + @Bind("domainJson") String domainJson, + @BindList("domainIds") List domainIds, + @Bind("after") long after, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE aboutFqnHash = :aboutFqnHash " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE aboutfqnhash = :aboutFqnHash " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByAbout( + @Bind("aboutFqnHash") String aboutFqnHash, + @Bind("after") long after, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE aboutFqnHash = :aboutFqnHash " + + "AND JSON_OVERLAPS(domains, :domainJson) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM activity_stream WHERE aboutfqnhash = :aboutFqnHash " + + "AND EXISTS (" + + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " + + "WHERE domain_id IN ()) " + + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", + connectionType = POSTGRES) + List listByAboutAndDomains( + @Bind("aboutFqnHash") String aboutFqnHash, + @Bind("domainJson") String domainJson, + @BindList("domainIds") List domainIds, + @Bind("after") long after, + @Bind("limit") int limit); + } + + interface ActivityStreamConfigDAO { + @ConnectionAwareSqlUpdate( + value = "INSERT INTO activity_stream_config(id, json) VALUES (:id, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO activity_stream_config(id, json) VALUES (:id, :json::jsonb)", + connectionType = POSTGRES) + void insert(@Bind("id") String id, @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "UPDATE activity_stream_config SET json = :json WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE activity_stream_config SET json = :json::jsonb WHERE id = :id", + connectionType = POSTGRES) + void update(@Bind("id") String id, @Bind("json") String json); + + @SqlQuery("SELECT json FROM activity_stream_config WHERE id = :id") + String findById(@Bind("id") String id); + + @ConnectionAwareSqlQuery( + value = "SELECT json FROM activity_stream_config WHERE domainId = :domainId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT json FROM activity_stream_config WHERE domainid = :domainId", + connectionType = POSTGRES) + String findByDomainId(@Bind("domainId") String domainId); + + @SqlQuery("SELECT json FROM activity_stream_config WHERE scope = 'global' LIMIT 1") + String findGlobalConfig(); + + @SqlQuery("SELECT json FROM activity_stream_config") + List listAll(); + + @SqlUpdate("DELETE FROM activity_stream_config WHERE id = :id") + void delete(@Bind("id") String id); + } + + @RegisterRowMapper(AuditLogRecordMapper.class) + interface AuditLogDAO { + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO audit_log_event(change_event_id, event_ts, event_type, user_name, " + + "actor_type, impersonated_by, service_name, " + + "entity_type, entity_id, entity_fqn, entity_fqn_hash, event_json, search_text, created_at) " + + "VALUES (:changeEventId::uuid, :eventTs, :eventType, :userName, " + + ":actorType, :impersonatedBy, :serviceName, " + + ":entityType, :entityId::uuid, :entityFQN, :entityFQNHash, :eventJson, :searchText, :createdAt) " + + "ON CONFLICT (change_event_id) DO NOTHING", + connectionType = POSTGRES) + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO audit_log_event(change_event_id, event_ts, event_type, user_name, " + + "actor_type, impersonated_by, service_name, " + + "entity_type, entity_id, entity_fqn, entity_fqn_hash, event_json, search_text, created_at) " + + "VALUES (:changeEventId, :eventTs, :eventType, :userName, " + + ":actorType, :impersonatedBy, :serviceName, " + + ":entityType, :entityId, :entityFQN, :entityFQNHash, :eventJson, :searchText, :createdAt)", + connectionType = MYSQL) + void insert(@BindBean AuditLogRecord record); + + @SqlQuery( + "SELECT id, change_event_id, event_ts, event_type, user_name, " + + "actor_type, impersonated_by, service_name, " + + "entity_type, entity_id, entity_fqn, entity_fqn_hash, event_json, search_text, created_at " + + "FROM audit_log_event LIMIT :limit") + List list( + @Define("condition") String condition, + @Define("orderClause") String orderClause, + @Bind("userName") String userName, + @Bind("actorType") String actorType, + @Bind("serviceName") String serviceName, + @Bind("entityType") String entityType, + @Bind("entityFQN") String entityFQN, + @Bind("entityFQNHASH") String entityFqnHash, + @Bind("eventType") String eventType, + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs, + @Bind("searchTerm") String searchTerm, + @Bind("afterEventTs") Long afterEventTs, + @Bind("afterId") Long afterId, + @Bind("limit") int limit); + + @SqlQuery("SELECT COUNT(id) FROM audit_log_event ") + int count( + @Define("condition") String condition, + @Bind("userName") String userName, + @Bind("actorType") String actorType, + @Bind("serviceName") String serviceName, + @Bind("entityType") String entityType, + @Bind("entityFQN") String entityFQN, + @Bind("entityFQNHASH") String entityFqnHash, + @Bind("eventType") String eventType, + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs, + @Bind("searchTerm") String searchTerm); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM audit_log_event " + + "WHERE created_at < :cutoffTs ORDER BY created_at LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM audit_log_event " + + "WHERE ctid IN ( " + + " SELECT ctid FROM audit_log_event " + + " WHERE created_at < :cutoffTs ORDER BY created_at LIMIT :limit " + + ")", + connectionType = POSTGRES) + int deleteInBatches(@Bind("cutoffTs") long cutoffTs, @Bind("limit") int limit); + } + + // OAuth 2.0 DAOs for MCP Server +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AiGovernanceDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AiGovernanceDAOs.java new file mode 100644 index 000000000000..2132f73fb077 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/AiGovernanceDAOs.java @@ -0,0 +1,343 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.util.List; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.util.jdbi.BindFQN; + +public interface AiGovernanceDAOs { + @CreateSqlObject + AIApplicationDAO aiApplicationDAO(); + + @CreateSqlObject + LLMModelDAO llmModelDAO(); + + @CreateSqlObject + PromptTemplateDAO promptTemplateDAO(); + + @CreateSqlObject + AgentExecutionDAO agentExecutionDAO(); + + @CreateSqlObject + AIGovernancePolicyDAO aiGovernancePolicyDAO(); + + @CreateSqlObject + McpServerDAO mcpServerDAO(); + + @CreateSqlObject + McpExecutionDAO mcpExecutionDAO(); + + @CreateSqlObject + LLMServiceDAO llmServiceDAO(); + + @CreateSqlObject + McpServiceDAO mcpServiceDAO(); + + interface AIApplicationDAO extends EntityDAO { + @Override + default String getTableName() { + return "ai_application_entity"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.ai.AIApplication.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface LLMModelDAO extends EntityDAO { + @Override + default String getTableName() { + return "llm_model_entity"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.ai.LLMModel.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface PromptTemplateDAO extends EntityDAO { + @Override + default String getTableName() { + return "prompt_template_entity"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.ai.PromptTemplate.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface AgentExecutionDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "agent_execution_entity"; + } + + @Override + default String getPartitionFieldName() { + return "agentId"; + } + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO agent_execution_entity(json) VALUES (:json) AS new_data ON DUPLICATE KEY UPDATE json = new_data.json", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO agent_execution_entity(json) VALUES (:json::jsonb) ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json", + connectionType = POSTGRES) + void insertWithoutExtension( + @Define("table") String table, + @BindFQN("entityFQNHash") String entityFQNHash, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO agent_execution_entity(json) VALUES (:json) AS new_data ON DUPLICATE KEY UPDATE json = new_data.json", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO agent_execution_entity(json) VALUES (:json::jsonb) ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json", + connectionType = POSTGRES) + void insert( + @Define("table") String table, + @BindFQN("entityFQNHash") String entityFQNHash, + @Bind("extension") String extension, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM agent_execution_entity WHERE agentId = :agentId AND timestamp = :timestamp", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM agent_execution_entity WHERE agentId = :agentId AND timestamp = :timestamp", + connectionType = POSTGRES) + void deleteAtTimestamp( + @BindFQN("agentId") String agentId, + @Bind("extension") String extension, + @Bind("timestamp") Long timestamp); + + @SqlQuery("SELECT count(*) FROM agent_execution_entity ") + int listCount(@Define("cond") String condition); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM agent_execution_entity " + + "WHERE id IN (" + + " SELECT id FROM (" + + " SELECT ae.id FROM agent_execution_entity ae " + + " LEFT JOIN ai_application_entity ai ON ae.agentId = ai.id " + + " WHERE ai.id IS NULL " + + " LIMIT :limit" + + " ) sub" + + ")", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM agent_execution_entity " + + "WHERE id IN (" + + " SELECT ae.id FROM agent_execution_entity ae " + + " LEFT JOIN ai_application_entity ai ON ae.agentId = ai.id " + + " WHERE ai.id IS NULL " + + " LIMIT :limit" + + ")", + connectionType = POSTGRES) + int deleteOrphanedRecords(@Bind("limit") int limit); + } + + interface AIGovernancePolicyDAO + extends EntityDAO { + @Override + default String getTableName() { + return "ai_governance_policy_entity"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.ai.AIGovernancePolicy.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface McpServerDAO extends EntityDAO { + @Override + default String getTableName() { + return "mcp_server_entity"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.ai.McpServer.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface McpExecutionDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "mcp_execution_entity"; + } + + @Override + default String getPartitionFieldName() { + return "serverId"; + } + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO (json) VALUES (:json) AS new_data ON DUPLICATE KEY UPDATE json = new_data.json", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO
(json) VALUES (:json::jsonb) ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json", + connectionType = POSTGRES) + void insertWithoutExtension( + @Define("table") String table, + @BindFQN("entityFQNHash") String entityFQNHash, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO
(json) VALUES (:json) AS new_data ON DUPLICATE KEY UPDATE json = new_data.json", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO
(json) VALUES (:json::jsonb) ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json", + connectionType = POSTGRES) + void insert( + @Define("table") String table, + @BindFQN("entityFQNHash") String entityFQNHash, + @Bind("extension") String extension, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "DELETE FROM
WHERE serverId = :serverId AND timestamp = :timestamp", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "DELETE FROM
WHERE serverId = :serverId AND timestamp = :timestamp", + connectionType = POSTGRES) + void deleteAtTimestamp( + @Define("table") String table, + @Bind("serverId") String serverId, + @Bind("extension") String extension, + @Bind("timestamp") Long timestamp); + + @SqlQuery( + "SELECT json FROM
WHERE serverId = :serverId ORDER BY timestamp DESC LIMIT :limit") + List listByServerId( + @Define("table") String table, @Bind("serverId") String serverId, @Bind("limit") int limit); + + @SqlQuery("SELECT count(*) FROM
") + int listCount(@Define("table") String table, @Define("cond") String condition); + + @ConnectionAwareSqlUpdate( + value = "DELETE FROM
WHERE serverId = :serverId", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "DELETE FROM
WHERE serverId = :serverId", + connectionType = POSTGRES) + void deleteByServerId(@Define("table") String table, @Bind("serverId") String serverId); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM mcp_execution_entity " + + "WHERE id IN (" + + " SELECT id FROM (" + + " SELECT me.id FROM mcp_execution_entity me " + + " LEFT JOIN mcp_server_entity ms ON me.serverId = ms.id " + + " WHERE ms.id IS NULL " + + " LIMIT :limit" + + " ) sub" + + ")", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM mcp_execution_entity " + + "WHERE id IN (" + + " SELECT me.id FROM mcp_execution_entity me " + + " LEFT JOIN mcp_server_entity ms ON me.serverId = ms.id " + + " WHERE ms.id IS NULL " + + " LIMIT :limit" + + ")", + connectionType = POSTGRES) + int deleteOrphanedRecords(@Bind("limit") int limit); + } + + interface LLMServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "llm_service_entity"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.services.LLMService.class; + } + } + + interface McpServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "mcp_service_entity"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.services.McpService.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BulkFieldFetcher.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BulkFieldFetcher.java new file mode 100644 index 000000000000..073689bace72 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BulkFieldFetcher.java @@ -0,0 +1,1271 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.Include.ALL; +import static org.openmetadata.schema.type.Include.NON_DELETED; +import static org.openmetadata.service.Entity.DATA_CONTRACT; +import static org.openmetadata.service.Entity.DATA_PRODUCT; +import static org.openmetadata.service.Entity.DOMAIN; +import static org.openmetadata.service.Entity.FIELD_CERTIFICATION; +import static org.openmetadata.service.Entity.FIELD_CHILDREN; +import static org.openmetadata.service.Entity.FIELD_DATA_CONTRACT; +import static org.openmetadata.service.Entity.FIELD_DATA_PRODUCTS; +import static org.openmetadata.service.Entity.FIELD_DOMAINS; +import static org.openmetadata.service.Entity.FIELD_EXPERTS; +import static org.openmetadata.service.Entity.FIELD_EXTENSION; +import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; +import static org.openmetadata.service.Entity.FIELD_OWNERS; +import static org.openmetadata.service.Entity.FIELD_REVIEWERS; +import static org.openmetadata.service.Entity.FIELD_TAGS; +import static org.openmetadata.service.Entity.FIELD_VOTES; +import static org.openmetadata.service.Entity.USER; +import static org.openmetadata.service.resources.tags.TagLabelUtil.populateTagLabel; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import jakarta.ws.rs.core.UriInfo; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.collections4.CollectionUtils; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.FieldInterface; +import org.openmetadata.schema.api.VoteRequest.VoteType; +import org.openmetadata.schema.system.EntityError; +import org.openmetadata.schema.type.AssetCertification; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.Votes; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.TypeRegistry; +import org.openmetadata.service.resources.tags.TagLabelUtil; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.FullyQualifiedName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.utils.Either; + +/** + * Bulk field-fetch helpers (the fetchAndSet and batchFetch families) for list reads, extracted + * from EntityRepository. Holds a back-reference to the owning repository (preserves polymorphism). + */ +final class BulkFieldFetcher { + private static final Logger LOG = LoggerFactory.getLogger(BulkFieldFetcher.class); + private final EntityRepository r; + + BulkFieldFetcher(EntityRepository r) { + this.r = r; + } + + Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields fields) { + if (nullOrEmpty(entities) || fields == null) { + return Collections.emptySet(); + } + + boolean loadOwners = r.supportsOwners && fields.contains(FIELD_OWNERS); + boolean loadFollowers = r.supportsFollower && fields.contains(FIELD_FOLLOWERS); + boolean loadDomains = r.supportsDomains && fields.contains(FIELD_DOMAINS); + boolean loadReviewers = r.supportsReviewers && fields.contains(FIELD_REVIEWERS); + boolean loadDataProducts = r.supportsDataProducts && fields.contains(FIELD_DATA_PRODUCTS); + boolean loadDataContract = r.supportsDataContract && fields.contains(FIELD_DATA_CONTRACT); + boolean loadVotes = r.supportsVotes && fields.contains(FIELD_VOTES); + boolean loadChildren = r.supportsChildren && fields.contains(FIELD_CHILDREN); + boolean loadExperts = r.supportsExperts && fields.contains(FIELD_EXPERTS); + + if (!loadOwners + && !loadFollowers + && !loadDomains + && !loadReviewers + && !loadDataProducts + && !loadDataContract + && !loadVotes + && !loadChildren + && !loadExperts) { + return Collections.emptySet(); + } + + List incomingRelations = new ArrayList<>(); + if (loadOwners) { + incomingRelations.add(Relationship.OWNS.ordinal()); + } + if (loadFollowers) { + incomingRelations.add(Relationship.FOLLOWS.ordinal()); + } + if (loadDomains || loadDataProducts) { + incomingRelations.add(Relationship.HAS.ordinal()); + } + if (loadReviewers) { + incomingRelations.add(Relationship.REVIEWS.ordinal()); + } + if (loadVotes) { + incomingRelations.add(Relationship.VOTED.ordinal()); + } + + List outgoingRelations = new ArrayList<>(); + if (loadChildren || loadDataContract) { + outgoingRelations.add(Relationship.CONTAINS.ordinal()); + } + if (loadExperts) { + outgoingRelations.add(Relationship.EXPERT.ordinal()); + } + + List entityIds = entityListToStrings(entities); + List incomingRecords = + incomingRelations.isEmpty() + ? Collections.emptyList() + : r.daoCollection + .relationshipDAO() + .findFromBatchWithRelations(entityIds, r.entityType, incomingRelations, ALL); + List outgoingRecords = + outgoingRelations.isEmpty() + ? Collections.emptyList() + : r.daoCollection + .relationshipDAO() + .findToBatchWithRelations(entityIds, r.entityType, outgoingRelations, ALL); + + Map> incomingRefsByType = + resolveRelationshipEntityReferencesByType(incomingRecords, true); + Map> outgoingRefsByType = + resolveRelationshipEntityReferencesByType(outgoingRecords, false); + + Map> ownersByEntity = loadOwners ? new HashMap<>() : null; + Map> followersByEntity = loadFollowers ? new HashMap<>() : null; + Map> domainsByEntity = loadDomains ? new HashMap<>() : null; + Map> reviewersByEntity = loadReviewers ? new HashMap<>() : null; + Map> dataProductsByEntity = + loadDataProducts ? new HashMap<>() : null; + Map> upVotersByEntity = loadVotes ? new HashMap<>() : null; + Map> downVotersByEntity = loadVotes ? new HashMap<>() : null; + Map> childrenByEntity = loadChildren ? new HashMap<>() : null; + Map dataContractByEntity = loadDataContract ? new HashMap<>() : null; + Map> expertsByEntity = loadExperts ? new HashMap<>() : null; + + for (CollectionDAO.EntityRelationshipObject record : incomingRecords) { + UUID entityId = UUID.fromString(record.getToId()); + UUID sourceId = UUID.fromString(record.getFromId()); + String sourceType = record.getFromEntity(); + EntityReference sourceRef = lookupRelationshipRef(incomingRefsByType, sourceType, sourceId); + if (sourceRef == null) { + continue; + } + + Relationship relationship = relationshipFromOrdinal(record.getRelation()); + if (relationship == null) { + continue; + } + switch (relationship) { + case OWNS -> { + if (loadOwners) { + ownersByEntity.computeIfAbsent(entityId, ignored -> new ArrayList<>()).add(sourceRef); + } + } + case FOLLOWS -> { + if (loadFollowers && USER.equals(sourceType)) { + followersByEntity + .computeIfAbsent(entityId, ignored -> new ArrayList<>()) + .add(sourceRef); + } + } + case HAS -> { + if (loadDomains && DOMAIN.equals(sourceType)) { + domainsByEntity.computeIfAbsent(entityId, ignored -> new ArrayList<>()).add(sourceRef); + } else if (loadDataProducts && DATA_PRODUCT.equals(sourceType)) { + dataProductsByEntity + .computeIfAbsent(entityId, ignored -> new ArrayList<>()) + .add(sourceRef); + } + } + case REVIEWS -> { + if (loadReviewers) { + reviewersByEntity + .computeIfAbsent(entityId, ignored -> new ArrayList<>()) + .add(sourceRef); + } + } + case VOTED -> { + if (loadVotes && USER.equals(sourceType)) { + VoteType voteType = JsonUtils.readValue(record.getJson(), VoteType.class); + if (voteType == VoteType.VOTED_UP) { + upVotersByEntity + .computeIfAbsent(entityId, ignored -> new ArrayList<>()) + .add(sourceRef); + } else if (voteType == VoteType.VOTED_DOWN) { + downVotersByEntity + .computeIfAbsent(entityId, ignored -> new ArrayList<>()) + .add(sourceRef); + } + } + } + default -> { + // no-op + } + } + } + + for (CollectionDAO.EntityRelationshipObject record : outgoingRecords) { + UUID entityId = UUID.fromString(record.getFromId()); + UUID targetId = UUID.fromString(record.getToId()); + String targetType = record.getToEntity(); + EntityReference targetRef = lookupRelationshipRef(outgoingRefsByType, targetType, targetId); + if (targetRef == null) { + continue; + } + + Relationship relationship = relationshipFromOrdinal(record.getRelation()); + if (relationship == null) { + continue; + } + switch (relationship) { + case CONTAINS -> { + if (loadChildren && r.entityType.equals(targetType)) { + childrenByEntity.computeIfAbsent(entityId, ignored -> new ArrayList<>()).add(targetRef); + } else if (loadDataContract && DATA_CONTRACT.equals(targetType)) { + dataContractByEntity.putIfAbsent(entityId, targetRef); + } + } + case EXPERT -> { + if (loadExperts && USER.equals(targetType)) { + expertsByEntity.computeIfAbsent(entityId, ignored -> new ArrayList<>()).add(targetRef); + } + } + default -> { + // no-op + } + } + } + + Set handledFields = new HashSet<>(); + for (T entity : entities) { + UUID entityId = entity.getId(); + if (loadOwners) { + entity.setOwners(ownersByEntity.getOrDefault(entityId, Collections.emptyList())); + } + if (loadFollowers) { + entity.setFollowers(followersByEntity.getOrDefault(entityId, Collections.emptyList())); + } + if (loadDomains) { + entity.setDomains(domainsByEntity.getOrDefault(entityId, Collections.emptyList())); + } + if (loadReviewers) { + entity.setReviewers(reviewersByEntity.getOrDefault(entityId, Collections.emptyList())); + } + if (loadDataProducts) { + entity.setDataProducts( + dataProductsByEntity.getOrDefault(entityId, Collections.emptyList())); + } + if (loadVotes) { + List upVoters = + upVotersByEntity.getOrDefault(entityId, Collections.emptyList()); + List downVoters = + downVotersByEntity.getOrDefault(entityId, Collections.emptyList()); + entity.setVotes( + new Votes() + .withUpVotes(upVoters.size()) + .withDownVotes(downVoters.size()) + .withUpVoters(upVoters) + .withDownVoters(downVoters)); + } + if (loadChildren) { + entity.setChildren(childrenByEntity.get(entityId)); + } + if (loadDataContract) { + entity.setDataContract(dataContractByEntity.get(entityId)); + } + if (loadExperts) { + entity.setExperts(expertsByEntity.getOrDefault(entityId, Collections.emptyList())); + } + } + + if (loadOwners) { + handledFields.add(FIELD_OWNERS); + } + if (loadFollowers) { + handledFields.add(FIELD_FOLLOWERS); + } + if (loadDomains) { + handledFields.add(FIELD_DOMAINS); + } + if (loadReviewers) { + handledFields.add(FIELD_REVIEWERS); + } + if (loadDataProducts) { + handledFields.add(FIELD_DATA_PRODUCTS); + } + if (loadVotes) { + handledFields.add(FIELD_VOTES); + } + if (loadChildren) { + handledFields.add(FIELD_CHILDREN); + } + if (loadDataContract) { + handledFields.add(FIELD_DATA_CONTRACT); + } + if (loadExperts) { + handledFields.add(FIELD_EXPERTS); + } + return handledFields; + } + + Map> resolveRelationshipEntityReferencesByType( + List records, boolean fromSide) { + if (records == null || records.isEmpty()) { + return Collections.emptyMap(); + } + + Map> idsByType = new HashMap<>(); + for (CollectionDAO.EntityRelationshipObject record : records) { + String entityTypeForRef = fromSide ? record.getFromEntity() : record.getToEntity(); + String entityId = fromSide ? record.getFromId() : record.getToId(); + if (nullOrEmpty(entityTypeForRef) + || nullOrEmpty(entityId) + || !Entity.hasEntityRepository(entityTypeForRef)) { + continue; + } + idsByType + .computeIfAbsent(entityTypeForRef, ignored -> new HashSet<>()) + .add(UUID.fromString(entityId)); + } + + if (idsByType.isEmpty()) { + return Collections.emptyMap(); + } + + Map> refsByType = new HashMap<>(); + for (Entry> entry : idsByType.entrySet()) { + List refs = + Entity.getEntityReferencesByIds( + entry.getKey(), new ArrayList<>(entry.getValue()), NON_DELETED); + refsByType.put( + entry.getKey(), + refs.stream() + .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a))); + } + return refsByType; + } + + EntityReference lookupRelationshipRef( + Map> refsByType, String entityType, UUID id) { + if (refsByType == null || nullOrEmpty(entityType) || id == null) { + return null; + } + Map refs = refsByType.get(entityType); + return refs == null ? null : refs.get(id); + } + + Relationship relationshipFromOrdinal(int relationOrdinal) { + Relationship[] values = Relationship.values(); + return relationOrdinal >= 0 && relationOrdinal < values.length ? values[relationOrdinal] : null; + } + + void fetchAndSetOwners(List entities, Fields fields) { + if (!fields.contains(FIELD_OWNERS) || !r.supportsOwners) { + return; + } + Map> ownersMap = batchFetchOwners(entities); + for (T entity : entities) { + entity.setOwners(ownersMap.getOrDefault(entity.getId(), Collections.emptyList())); + } + } + + void fetchAndSetFollowers(List entities, Fields fields) { + if (!fields.contains(FIELD_FOLLOWERS) || !r.supportsFollower) { + return; + } + Map> followersMap = batchFetchFollowers(entities); + for (T entity : entities) { + entity.setFollowers(followersMap.getOrDefault(entity.getId(), Collections.emptyList())); + } + } + + void fetchAndSetTags(List entities, Fields fields) { + if (!fields.contains(FIELD_TAGS) || !r.supportsTags) { + return; + } + + List entityFQNs = + entities.stream().map(EntityInterface::getFullyQualifiedName).toList(); + + Map> tagsMap = batchFetchTags(entityFQNs); + + // Batch fetch all derived tags in ONE query instead of N queries + List allTags = + tagsMap.values().stream().flatMap(List::stream).collect(Collectors.toList()); + Map> derivedTagsMap = TagLabelUtil.batchFetchDerivedTags(allTags); + + for (T entity : entities) { + List entityTags = + tagsMap.getOrDefault(entity.getFullyQualifiedName(), Collections.emptyList()); + entity.setTags(TagLabelUtil.addDerivedTagsWithPreFetched(entityTags, derivedTagsMap)); + } + } + + void fetchAndSetDomains(List entities, Fields fields) { + if (!fields.contains(FIELD_DOMAINS) || !r.supportsDomains) { + return; + } + + Map> domainsMap = batchFetchDomains(entities); + + for (T entity : entities) { + entity.setDomains(domainsMap.getOrDefault(entity.getId(), Collections.emptyList())); + } + } + + void fetchAndSetExtension(List entities, Fields fields) { + if (!fields.contains(FIELD_EXTENSION) + || !r.supportsExtension + || entities == null + || entities.isEmpty()) { + return; + } + + Map extensionsMap = batchFetchExtensions(entities); + + for (T entity : entities) { + Object extension = extensionsMap.get(entity.getId()); + entity.setExtension(extension); + } + } + + void fetchAndSetChildren(List entities, Fields fields) { + if (!fields.contains(FIELD_CHILDREN) || entities == null || nullOrEmpty(entities)) { + return; + } + + Map> childrenMap = batchFetchChildren(entities); + + for (T entity : entities) { + entity.setChildren(childrenMap.get(entity.getId())); + } + } + + void fetchAndSetExperts(List entities, Fields fields) { + if (!fields.contains(FIELD_EXPERTS) || !r.supportsExperts || nullOrEmpty(entities)) { + return; + } + + Map> expertsMap = batchFetchExperts(entities); + + for (T entity : entities) { + entity.setExperts(expertsMap.getOrDefault(entity.getId(), Collections.emptyList())); + } + } + + void fetchAndSetReviewers(List entities, Fields fields) { + if (!fields.contains(FIELD_REVIEWERS) || !r.supportsReviewers || nullOrEmpty(entities)) { + return; + } + + Map> reviewersMap = batchFetchReviewers(entities); + + for (T entity : entities) { + List reviewers = + reviewersMap.getOrDefault(entity.getId(), Collections.emptyList()); + entity.setReviewers(reviewers); + } + } + + void fetchAndSetVotes(List entities, Fields fields) { + if (!fields.contains(FIELD_VOTES) || !r.supportsVotes || nullOrEmpty(entities)) { + return; + } + + Map votesMap = batchFetchVotes(entities); + + for (T entity : entities) { + entity.setVotes(votesMap.getOrDefault(entity.getId(), new Votes())); + } + } + + void enrichEntitiesForAuth(List entities) { + if (entities == null || entities.isEmpty()) return; + Map> ownersMap = batchFetchOwners(entities); + Map> domainsMap = batchFetchDomains(entities); + for (T entity : entities) { + entity.setOwners(ownersMap.getOrDefault(entity.getId(), entity.getOwners())); + entity.setDomains(domainsMap.getOrDefault(entity.getId(), entity.getDomains())); + } + } + + void fetchAndSetDataProducts(List entities, Fields fields) { + if (!fields.contains(FIELD_DATA_PRODUCTS) || !r.supportsDataProducts || nullOrEmpty(entities)) { + return; + } + + Map> dataProductsMap = batchFetchDataProducts(entities); + + for (T entity : entities) { + entity.setDataProducts(dataProductsMap.getOrDefault(entity.getId(), Collections.emptyList())); + } + } + + void fetchAndSetCertification(List entities, Fields fields) { + if (!fields.contains(FIELD_CERTIFICATION) + || !r.supportsCertification + || nullOrEmpty(entities)) { + return; + } + + Map certificationMap = batchFetchCertification(entities); + + for (T entity : entities) { + entity.setCertification(certificationMap.get(entity.getId())); + } + } + + Map> batchFetchOwners(List entities) { + var ownersMap = new HashMap>(); + + if (entities == null || entities.isEmpty()) { + return ownersMap; + } + // Use Include.ALL to get all relationships including those for soft-deleted entities + // Use the 3-parameter version to find owners of any entity type (equivalent to passing null in + // single entity version) + var records = + r.daoCollection + .relationshipDAO() + .findFromBatch(entityListToStrings(entities), Relationship.OWNS.ordinal(), ALL); + + LOG.debug( + "batchFetchOwners: Found {} owner relationships for {} entities", + records.size(), + entities.size()); + + // Cache UUID conversions to avoid repeated parsing + Map uuidCache = new HashMap<>(); + + // Group records by entity type to batch fetch entity references (with deduplication) + var ownerIdsByType = new HashMap>(); + records.forEach( + rec -> { + var fromEntity = rec.getFromEntity(); + var fromId = uuidCache.computeIfAbsent(rec.getFromId(), UUID::fromString); + ownerIdsByType.computeIfAbsent(fromEntity, k -> new HashSet<>()).add(fromId); + }); + + // Batch fetch entity references for each entity type + var ownerRefsByType = new HashMap>(); + ownerIdsByType.forEach( + (entityType, ownerIds) -> { + var ownerRefs = + Entity.getEntityReferencesByIds(entityType, new ArrayList<>(ownerIds), NON_DELETED); + var refMap = + ownerRefs.stream() + .collect(Collectors.toMap(EntityReference::getId, ref -> ref, (a, b) -> a)); + ownerRefsByType.put(entityType, refMap); + }); + + // Map owners to entities (reuse cached UUIDs) + records.forEach( + rec -> { + var toId = uuidCache.computeIfAbsent(rec.getToId(), UUID::fromString); + var fromId = uuidCache.get(rec.getFromId()); // Already cached + var fromEntity = rec.getFromEntity(); + + var refMap = ownerRefsByType.get(fromEntity); + if (refMap != null) { + var ownerRef = refMap.get(fromId); + if (ownerRef != null) { + ownersMap.computeIfAbsent(toId, k -> new ArrayList<>()).add(ownerRef); + } + } + }); + + return ownersMap; + } + + Map> batchFetchFollowers(List entities) { + if (entities == null || entities.isEmpty()) { + return Collections.emptyMap(); + } + + List records = + r.daoCollection + .relationshipDAO() + .findFromBatch( + entityListToStrings(entities), Relationship.FOLLOWS.ordinal(), Include.ALL); + + Map> followersMap = new HashMap<>(); + + List followerIds = + records.stream() + .map(record -> UUID.fromString(record.getFromId())) + .collect(Collectors.toList()); + + Map followerRefs = + Entity.getEntityReferencesByIds(USER, followerIds, NON_DELETED).stream() + .collect(Collectors.toMap(EntityReference::getId, Function.identity())); + + records.forEach( + record -> { + UUID entityId = UUID.fromString(record.getToId()); + UUID followerId = UUID.fromString(record.getFromId()); + EntityReference followerRef = followerRefs.get(followerId); + if (followerRef != null) { + followersMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(followerRef); + } + }); + + return followersMap; + } + + Map batchFetchVotes(List entities) { + var votesMap = new HashMap(); + if (entities == null || entities.isEmpty()) { + return votesMap; + } + + var records = + r.daoCollection + .relationshipDAO() + .findFromBatch( + entityListToStrings(entities), Relationship.VOTED.ordinal(), Entity.USER, ALL); + + var upVoterIds = new HashMap>(); + var downVoterIds = new HashMap>(); + records.forEach( + rec -> { + UUID entityId = UUID.fromString(rec.getToId()); + UUID userId = UUID.fromString(rec.getFromId()); + VoteType type = JsonUtils.readValue(rec.getJson(), VoteType.class); + if (type == VoteType.VOTED_UP) { + upVoterIds.computeIfAbsent(entityId, k -> new ArrayList<>()).add(userId); + } else if (type == VoteType.VOTED_DOWN) { + downVoterIds.computeIfAbsent(entityId, k -> new ArrayList<>()).add(userId); + } + }); + + Set allUserIds = new HashSet<>(); + upVoterIds.values().forEach(allUserIds::addAll); + downVoterIds.values().forEach(allUserIds::addAll); + Map userRefs = + Entity.getEntityReferencesByIds(Entity.USER, new ArrayList<>(allUserIds), NON_DELETED) + .stream() + .collect(Collectors.toMap(EntityReference::getId, Function.identity())); + + for (T entity : entities) { + List up = + upVoterIds.getOrDefault(entity.getId(), Collections.emptyList()).stream() + .map(userRefs::get) + .filter(Objects::nonNull) + .toList(); + List down = + downVoterIds.getOrDefault(entity.getId(), Collections.emptyList()).stream() + .map(userRefs::get) + .filter(Objects::nonNull) + .toList(); + votesMap.put( + entity.getId(), + new Votes() + .withUpVotes(up.size()) + .withDownVotes(down.size()) + .withUpVoters(up) + .withDownVoters(down)); + } + + return votesMap; + } + + Map> batchFetchDataProducts(List entities) { + return batchFetchToIdsOneToMany(entities, Relationship.HAS, Entity.DATA_PRODUCT); + } + + Map batchFetchCertification(List entities) { + var result = new HashMap(); + if (entities == null || entities.isEmpty() || !r.supportsCertification) { + return result; + } + + long startTime = System.currentTimeMillis(); + + String certClassification = r.getCertificationClassification(); + if (certClassification == null) { + return result; + } + + // Build FQN hash → entity ID map (batch query uses hashed FQNs for lookup) + Map entityIdByFqnHash = new HashMap<>(); + List fqnList = new ArrayList<>(); + for (T entity : entities) { + fqnList.add(entity.getFullyQualifiedName()); + entityIdByFqnHash.put( + FullyQualifiedName.buildHash(entity.getFullyQualifiedName()), entity.getId()); + } + + List certTags; + try { + certTags = + r.daoCollection + .tagUsageDAO() + .getCertTagsInternalBatch( + TagLabel.TagSource.CLASSIFICATION.ordinal(), + fqnList, + FullyQualifiedName.buildHash(certClassification) + ".%"); + } catch (Exception e) { + LOG.warn( + "batchFetchCertification: batch query failed, falling back to individual fetch: {}", + e.getMessage()); + for (T entity : entities) { + result.put(entity.getId(), r.getCertification(entity)); + } + return result; + } + + for (CollectionDAO.TagUsageDAO.TagLabelWithFQNHash tagWithHash : certTags) { + UUID entityId = entityIdByFqnHash.get(tagWithHash.getTargetFQNHash()); + if (entityId == null || result.containsKey(entityId)) { + continue; + } + TagLabel tagLabel = tagWithHash.toTagLabel(); + TagLabelUtil.applyTagCommonFieldsGracefully(tagLabel); + result.put( + entityId, + new AssetCertification() + .withTagLabel(tagLabel) + .withAppliedDate( + tagLabel.getAppliedAt() != null ? tagLabel.getAppliedAt().getTime() : null) + .withExpiryDate( + tagLabel.getMetadata() != null ? tagLabel.getMetadata().getExpiryDate() : null)); + } + + LOG.debug( + "batchFetchCertification: {} entities, {} certs found in {}ms", + entities.size(), + result.size(), + System.currentTimeMillis() - startTime); + + return result; + } + + /** + * Creates a unique key for a TagLabel combining TagFQN and Source for fast Set-based lookups. + * This replaces O(n) stream().anyMatch() operations with O(1) Set.contains() operations. + */ + String createTagKey(TagLabel tag) { + return tag.getTagFQN() + ":" + tag.getSource(); + } + + /** + * Creates a Set of tag keys from a list of TagLabels for efficient O(1) lookups. + */ + Set createTagKeySet(List tags) { + return tags.stream().map(this::createTagKey).collect(Collectors.toSet()); + } + + Map> batchFetchTags(List entityFQNs) { + if (entityFQNs == null || entityFQNs.isEmpty()) { + return Collections.emptyMap(); + } + + Map> targetHashToTagLabel = + populateTagLabel( + listOrEmpty(r.daoCollection.tagUsageDAO().getTagsInternalBatch(entityFQNs))); + String certClassification = r.getCertificationClassification(); + return entityFQNs.stream() + .collect( + Collectors.toMap( + Function.identity(), + fqn -> { + String targetFQNHash = FullyQualifiedName.buildHash(fqn); + List tags = + Optional.ofNullable(targetHashToTagLabel.get(targetFQNHash)) + .filter(list -> !list.isEmpty()) + .orElseGet(ArrayList::new); + if (certClassification != null) { + tags.removeIf( + tag -> + certClassification.equals( + FullyQualifiedName.getParentFQN(tag.getTagFQN()))); + } + return tags; + }, + (a, b) -> a)); + } + + Map> batchFetchDomains(List entities) { + Map> domainsMap = new HashMap<>(); + + if (entities == null || entities.isEmpty()) { + return domainsMap; + } + List records = + r.daoCollection + .relationshipDAO() + .findFromBatch(entityListToStrings(entities), Relationship.HAS.ordinal(), DOMAIN, ALL); + + // Collect all unique domain IDs first + var domainIds = + records.stream().map(rec -> UUID.fromString(rec.getFromId())).distinct().toList(); + + // Batch fetch all domain entity references + var domainRefs = Entity.getEntityReferencesByIds(DOMAIN, domainIds, ALL); + var domainRefMap = + domainRefs.stream().collect(Collectors.toMap(EntityReference::getId, ref -> ref)); + + for (CollectionDAO.EntityRelationshipObject rec : records) { + UUID toId = UUID.fromString(rec.getToId()); + UUID fromId = UUID.fromString(rec.getFromId()); + EntityReference domainRef = domainRefMap.get(fromId); + domainsMap.computeIfAbsent(toId, k -> new ArrayList<>()).add(domainRef); + } + + return domainsMap; + } + + Map> batchFetchReviewers(List entities) { + if (entities == null || entities.isEmpty()) { + return new HashMap<>(); + } + + // Use Include.ALL to get all relationships including those for soft-deleted entities + var records = + r.daoCollection + .relationshipDAO() + .findFromBatch(entityListToStrings(entities), Relationship.REVIEWS.ordinal(), ALL); + + var reviewersMap = new HashMap>(); + + // Cache UUID conversions to avoid repeated parsing + Map uuidCache = new HashMap<>(); + + // Group records by entity type to batch fetch entity references (with deduplication) + var reviewerIdsByType = new HashMap>(); + records.forEach( + rec -> { + var fromEntity = rec.getFromEntity(); + var fromId = uuidCache.computeIfAbsent(rec.getFromId(), UUID::fromString); + reviewerIdsByType.computeIfAbsent(fromEntity, k -> new HashSet<>()).add(fromId); + }); + + // Batch fetch entity references for each entity type + var reviewerRefsByType = new HashMap>(); + reviewerIdsByType.forEach( + (entityType, reviewerIds) -> { + var reviewerRefs = + Entity.getEntityReferencesByIds( + entityType, new ArrayList<>(reviewerIds), NON_DELETED); + var refMap = + reviewerRefs.stream() + .collect(Collectors.toMap(EntityReference::getId, ref -> ref, (a, b) -> a)); + reviewerRefsByType.put(entityType, refMap); + }); + + // Map reviewers to entities (reuse cached UUIDs) + records.forEach( + rec -> { + var entityId = uuidCache.computeIfAbsent(rec.getToId(), UUID::fromString); + var fromId = uuidCache.get(rec.getFromId()); // Already cached + var fromEntity = rec.getFromEntity(); + + var refMap = reviewerRefsByType.get(fromEntity); + if (refMap != null) { + var reviewerRef = refMap.get(fromId); + if (reviewerRef != null) { + reviewersMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(reviewerRef); + } + } + }); + + return reviewersMap; + } + + Map batchFetchExtensions(List entities) { + if (!r.supportsExtension || entities == null || entities.isEmpty()) { + return Collections.emptyMap(); + } + String fieldFQNPrefix = TypeRegistry.getCustomPropertyFQNPrefix(r.entityType); + + List records = + r.daoCollection + .entityExtensionDAO() + .getExtensionsBatch(entityListToStrings(entities), fieldFQNPrefix); + + Map> extensionsMap = + records.stream().collect(Collectors.groupingBy(CollectionDAO.ExtensionRecordWithId::id)); + + Map result = new HashMap<>(); + + for (Entry> entry : extensionsMap.entrySet()) { + UUID entityId = entry.getKey(); + List extensionRecords = entry.getValue(); + + ObjectNode objectNode = JsonUtils.getObjectNode(); + for (CollectionDAO.ExtensionRecordWithId record : extensionRecords) { + String fieldName = TypeRegistry.getPropertyName(record.extensionName()); + JsonNode extensionJsonNode = JsonUtils.readTree(record.extensionJson()); + objectNode.set(fieldName, extensionJsonNode); + } + + result.put(entityId, objectNode); + } + + return result; + } + + Map> batchFetchExperts(List entities) { + if (!r.supportsExperts || nullOrEmpty(entities)) { + return Collections.emptyMap(); + } + + // Batch fetch all expert relationships - experts are TO relationships + List records = + r.daoCollection + .relationshipDAO() + .findToBatch(entityListToStrings(entities), Relationship.EXPERT.ordinal(), USER); + + Map> expertsMap = new HashMap<>(); + + // Cache UUID conversions to avoid repeated parsing + Map uuidCache = new HashMap<>(); + + // findToBatch returns fromId=entity, toId=user — collect user IDs from toId + List expertIds = + records.stream() + .map(record -> uuidCache.computeIfAbsent(record.getToId(), UUID::fromString)) + .distinct() + .collect(Collectors.toList()); + + // Batch fetch all expert references, filtering out soft-deleted users + Map expertRefs = + Entity.getEntityReferencesByIds(USER, expertIds, NON_DELETED).stream() + .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); + + // Group experts by entity + records.forEach( + record -> { + UUID entityId = uuidCache.computeIfAbsent(record.getFromId(), UUID::fromString); + UUID expertId = uuidCache.get(record.getToId()); // Already cached above + EntityReference expertRef = expertRefs.get(expertId); + if (expertRef != null) { + expertsMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(expertRef); + } + }); + + LOG.debug( + "batchFetchExperts: Found {} expert relationships for {} entities", + records.size(), + entities.size()); + + return expertsMap; + } + + Map> batchFetchChildren(List entities) { + if (entities == null || entities.isEmpty()) { + return new HashMap<>(); + } + + // Use Include.ALL to get all relationships including those for soft-deleted entities + var records = + r.daoCollection + .relationshipDAO() + .findToBatch( + entityListToStrings(entities), Relationship.CONTAINS.ordinal(), r.entityType, ALL); + + var childrenMap = new HashMap>(); + + if (CollectionUtils.isEmpty(records)) { + return childrenMap; + } + + var idReferenceMap = + Entity.getEntityReferencesByIds( + records.get(0).getToEntity(), + records.stream().map(e -> UUID.fromString(e.getToId())).distinct().toList(), + ALL) + .stream() + .collect(Collectors.toMap(e -> e.getId().toString(), Function.identity())); + + records.forEach( + rec -> { + var entityId = UUID.fromString(rec.getFromId()); + var childrenRef = idReferenceMap.get(rec.getToId()); + if (childrenRef != null) { + childrenMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(childrenRef); + } + }); + + return childrenMap; + } + + List entityListToStrings(List entities) { + return entities.stream().map(EntityInterface::getId).map(UUID::toString).toList(); + } + + Iterator> serializeJsons( + List jsons, Fields fields, UriInfo uriInfo) { + List> results = new ArrayList<>(); + List entities = new ArrayList<>(); + + for (String json : jsons) { + try { + T entity = JsonUtils.readValue(json, r.entityClass); + entities.add(entity); + } catch (Exception e) { + EntityError entityError = + new EntityError() + .withMessage("Failed to deserialize entity: " + e.getMessage()) + .withEntity(null); + results.add(Either.right(entityError)); + } + } + + if (!entities.isEmpty()) { + try { + r.setFieldsInBulk(fields, entities); + if (!nullOrEmpty(uriInfo)) { + entities.forEach(entity -> r.withHref(uriInfo, entity)); + } + + for (T entity : entities) { + results.add(Either.left(entity)); + } + } catch (Exception e) { + LOG.warn("setFieldsInBulk failed in serializeJsons, falling back to per-entity loading", e); + for (T entity : entities) { + try { + r.setFieldsInternal(entity, fields); + r.setInheritedFields(entity, fields); + r.clearFieldsInternal(entity, fields); + if (!nullOrEmpty(uriInfo)) { + entity = r.withHref(uriInfo, entity); + } + results.add(Either.left(entity)); + } catch (Exception individualError) { + r.clearFieldsInternal(entity, fields); + EntityError entityError = + new EntityError().withMessage(individualError.getMessage()).withEntity(entity); + results.add(Either.right(entityError)); + } + } + } + } + return results.iterator(); + } + + void setFieldFromMap( + boolean includeField, List entities, Map valueMap, BiConsumer setter) { + if (!includeField || entities.isEmpty()) { + return; + } + for (T entity : entities) { + V value = valueMap.get(entity.getId()); + setter.accept(entity, value); + } + } + + void setFieldFromMapSingleRelation( + boolean includeField, List entities, Map valueMap, BiConsumer setter) { + if (!includeField || entities.isEmpty()) { + return; + } + for (T entity : entities) { + V value = valueMap.get(entity.getId()); + setter.accept(entity, value); + } + } + + Map> batchFetchFromIdsManyToOne( + List entities, Relationship relationship, String toEntityType) { + var resultMap = new HashMap>(); + if (entities == null || entities.isEmpty()) { + return resultMap; + } + + // Use Include.ALL to get all relationships including those for soft-deleted entities + var records = + r.daoCollection + .relationshipDAO() + .findToBatch(entityListToStrings(entities), relationship.ordinal(), toEntityType, ALL); + + var idReferenceMap = + Entity.getEntityReferencesByIds( + toEntityType, + records.stream().map(e -> UUID.fromString(e.getToId())).distinct().toList(), + ALL) + .stream() + .collect(Collectors.toMap(e -> e.getId().toString(), Function.identity())); + + records.forEach( + record -> { + var entityId = UUID.fromString(record.getFromId()); + var relatedRef = idReferenceMap.get(record.getToId()); + if (relatedRef != null) { + resultMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(relatedRef); + } + }); + + return resultMap; + } + + Map> batchFetchToIdsOneToMany( + List entities, Relationship relationship, String fromEntityType) { + var resultMap = new HashMap>(); + if (entities == null || entities.isEmpty()) { + return resultMap; + } + + // Use Include.ALL to get all relationships including those for soft-deleted entities + var records = + r.daoCollection + .relationshipDAO() + .findFromBatch( + entityListToStrings(entities), relationship.ordinal(), fromEntityType, ALL); + + var idReferenceMap = + Entity.getEntityReferencesByIds( + fromEntityType, + records.stream().map(e -> UUID.fromString(e.getFromId())).distinct().toList(), + ALL) + .stream() + .collect(Collectors.toMap(e -> e.getId().toString(), Function.identity())); + + records.forEach( + record -> { + var entityId = UUID.fromString(record.getToId()); + var relatedRef = idReferenceMap.get(record.getFromId()); + if (relatedRef != null) { + resultMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(relatedRef); + } + }); + + return resultMap; + } + + Map batchFetchFromIdsAndRelationSingleRelation( + List entities, Relationship relationship) { + var resultMap = new HashMap(); + if (entities == null || entities.isEmpty()) { + return resultMap; + } + + // Use Include.ALL to get all relationships including those for soft-deleted entities + var records = + r.daoCollection + .relationshipDAO() + .findFromBatch(entityListToStrings(entities), relationship.ordinal(), ALL); + + var idReferenceMap = new HashMap(); + + // Group by entity type to make efficient batch calls + var entityTypeToIds = + records.stream() + .collect( + Collectors.groupingBy( + CollectionDAO.EntityRelationshipObject::getFromEntity, + Collectors.mapping( + CollectionDAO.EntityRelationshipObject::getFromId, Collectors.toList()))); + + entityTypeToIds.forEach( + (entityType, idStrings) -> { + var ids = idStrings.stream().map(UUID::fromString).distinct().toList(); + var refs = Entity.getEntityReferencesByIds(entityType, ids, ALL); + refs.forEach(ref -> idReferenceMap.put(ref.getId().toString(), ref)); + }); + + records.forEach( + record -> { + var entityId = UUID.fromString(record.getToId()); + var relatedRef = idReferenceMap.get(record.getFromId()); + if (relatedRef != null) { + resultMap.put(entityId, relatedRef); + } + }); + + return resultMap; + } + + /** Bulk populate field tags for multiple entities using chunked exact-match IN on field FQN hashes. */ + void bulkPopulateEntityFieldTags( + List entities, java.util.function.Function> fieldExtractor) { + + if (entities == null || entities.isEmpty()) { + return; + } + + Set fieldFQNs = new LinkedHashSet<>(); + Map> flatFieldsByEntity = new HashMap<>(); + for (T entity : entities) { + List fields = fieldExtractor.apply(entity); + if (fields != null) { + List flattenedFields = EntityUtil.getFlattenedEntityField(fields); + flatFieldsByEntity.put(entity, flattenedFields); + for (F field : listOrEmpty(flattenedFields)) { + if (field.getFullyQualifiedName() != null) { + fieldFQNs.add(field.getFullyQualifiedName()); + } + } + } + } + + if (fieldFQNs.isEmpty()) { + return; + } + + // Fetch tags in chunked IN queries, then enrich once + List fieldFQNList = new ArrayList<>(fieldFQNs); + int batchSize = 5000; + List tagUsages = new ArrayList<>(); + for (int i = 0; i < fieldFQNList.size(); i += batchSize) { + List chunk = fieldFQNList.subList(i, Math.min(i + batchSize, fieldFQNList.size())); + tagUsages.addAll(listOrEmpty(r.daoCollection.tagUsageDAO().getTagsInternalBatch(chunk))); + } + Map> tagsByFieldHash = populateTagLabel(tagUsages); + + Map> derivedTagsMap; + try { + List tagLabels = + tagsByFieldHash.values().stream().flatMap(List::stream).collect(Collectors.toList()); + derivedTagsMap = TagLabelUtil.batchFetchDerivedTags(tagLabels); + } catch (Exception ex) { + LOG.warn("Failed to batch fetch derived tags for fields. Skipping derived tags.", ex); + derivedTagsMap = Collections.emptyMap(); + } + + for (T entity : entities) { + List flattenedFields = flatFieldsByEntity.get(entity); + if (flattenedFields != null) { + for (F field : listOrEmpty(flattenedFields)) { + String fieldHash = FullyQualifiedName.buildHash(field.getFullyQualifiedName()); + List fieldTags = tagsByFieldHash.get(fieldHash); + if (fieldTags == null) { + field.setTags(new ArrayList<>()); + } else { + field.setTags(TagLabelUtil.addDerivedTagsWithPreFetched(fieldTags, derivedTagsMap)); + } + } + } + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BulkImportService.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BulkImportService.java new file mode 100644 index 000000000000..e604dbcd69f0 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/BulkImportService.java @@ -0,0 +1,1078 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.EventType.ENTITY_CREATED; +import static org.openmetadata.schema.type.EventType.ENTITY_NO_CHANGE; +import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED; +import static org.openmetadata.service.monitoring.RequestLatencyContext.phase; + +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Timer; +import jakarta.ws.rs.core.Response.Status; +import jakarta.ws.rs.core.UriInfo; +import java.io.IOException; +import java.io.StringWriter; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.CSVRecord; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.type.ApiStatus; +import org.openmetadata.schema.type.ChangeEvent; +import org.openmetadata.schema.type.EventType; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.api.BulkDeleteStaleRequest; +import org.openmetadata.schema.type.api.BulkOperationResult; +import org.openmetadata.schema.type.api.BulkResponse; +import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.BadRequestException; +import org.openmetadata.service.formatter.util.FormatterUtil; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.RestUtil.PutResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Async/sequential bulk create-update-delete pipeline, CSV bulk-import result shaping, and + * stale-entity deletion. Extracted from EntityRepository; holds a back-reference (r). + */ +final class BulkImportService { + private static final Logger LOG = LoggerFactory.getLogger(BulkImportService.class); + private final EntityRepository r; + + BulkImportService(EntityRepository r) { + this.r = r; + } + + CsvImportResult createLeanCsvImportResult(CsvImportResult fullResult) { + CsvImportResult leanResult = + new CsvImportResult() + .withDryRun(fullResult.getDryRun()) + .withStatus(fullResult.getStatus()) + .withNumberOfRowsProcessed(fullResult.getNumberOfRowsProcessed()) + .withNumberOfRowsPassed(fullResult.getNumberOfRowsPassed()) + .withNumberOfRowsFailed(fullResult.getNumberOfRowsFailed()) + .withAbortReason(fullResult.getAbortReason()); + + if (nullOrEmpty(fullResult.getImportResultsCsv())) { + return leanResult; + } + + StringWriter stringWriter = new StringWriter(); + try (CSVParser parser = + CSVParser.parse( + fullResult.getImportResultsCsv(), CSVFormat.DEFAULT.withFirstRecordAsHeader())) { + int nameIndex = -1; + List headerNames = parser.getHeaderNames(); + for (int i = 0; i < headerNames.size(); i++) { + if (headerNames.get(i).toLowerCase().contains("name")) { + nameIndex = i; + break; + } + } + + String[] leanHeaders = {"status", "details", "name"}; + try (CSVPrinter printer = + new CSVPrinter(stringWriter, CSVFormat.DEFAULT.withHeader(leanHeaders))) { + for (CSVRecord record : parser) { + String name = (nameIndex != -1 && nameIndex < record.size()) ? record.get(nameIndex) : ""; + printer.printRecord(record.get("status"), record.get("details"), name); + } + } + leanResult.setImportResultsCsv(stringWriter.toString()); + } catch (IOException | IllegalArgumentException e) { + // If parsing fails, just return the original CSV to avoid losing data + LOG.warn("Failed to create lean CSV for change description, returning full CSV", e); + leanResult.setImportResultsCsv(fullResult.getImportResultsCsv()); + } + return leanResult; + } + + static final ConcurrentHashMap> BULK_JOBS = + new ConcurrentHashMap<>(); + + // Cached metrics to avoid Timer.builder overhead on every call + static final ConcurrentHashMap ENTITY_LATENCY_TIMERS = new ConcurrentHashMap<>(); + static final ConcurrentHashMap ENTITY_QUEUE_WAIT_TIMERS = + new ConcurrentHashMap<>(); + static final ConcurrentHashMap BULK_OPERATION_TIMERS = new ConcurrentHashMap<>(); + static final ConcurrentHashMap BATCH_SIZE_SUMMARIES = + new ConcurrentHashMap<>(); + static final ConcurrentHashMap SUCCESS_RATE_SUMMARIES = + new ConcurrentHashMap<>(); + + static final int MAX_CONCURRENT_BULK_JOBS = 100; + static final Semaphore BULK_JOB_PERMITS = new Semaphore(MAX_CONCURRENT_BULK_JOBS); + + CompletableFuture submitAsyncBulkOperation( + UriInfo uriInfo, + List entities, + String userName, + Map existingByFqn, + boolean overrideMetadata, + List authFailedResponses, + int totalRequests) { + + // Acquire a permit before scheduling — Semaphore is thread-safe and avoids TOCTOU races + if (!BULK_JOB_PERMITS.tryAcquire()) { + throw new jakarta.ws.rs.WebApplicationException( + "Too many concurrent bulk jobs (max " + MAX_CONCURRENT_BULK_JOBS + "). Retry later.", + jakarta.ws.rs.core.Response.Status.TOO_MANY_REQUESTS); + } + + String jobId = UUID.randomUUID().toString(); + LOG.info( + "Submitting async bulk operation with jobId: {} for {} entities", jobId, entities.size()); + + CompletableFuture job; + try { + job = + CompletableFuture.supplyAsync( + () -> { + try { + return bulkCreateOrUpdateEntitiesSequential( + uriInfo, entities, userName, existingByFqn, overrideMetadata); + } catch (Exception e) { + LOG.error("Async bulk operation failed for jobId: {}", jobId, e); + BulkOperationResult errorResult = new BulkOperationResult(); + errorResult.setStatus(ApiStatus.FAILURE); + errorResult.setNumberOfRowsFailed(entities.size()); + errorResult.setNumberOfRowsPassed(0); + return errorResult; + } + }, + BulkExecutor.getInstance().getExecutor()); + } catch (Exception e) { + BULK_JOB_PERMITS.release(); + throw e; + } + + // Merge auth failures into the final result so polling clients see the complete picture + CompletableFuture mergedJob = + job.thenApply( + result -> { + if (!authFailedResponses.isEmpty()) { + result.setNumberOfRowsFailed( + result.getNumberOfRowsFailed() + authFailedResponses.size()); + result.setNumberOfRowsProcessed(totalRequests); + if (result.getFailedRequest() == null) { + result.setFailedRequest(new ArrayList<>(authFailedResponses)); + } else { + result.getFailedRequest().addAll(authFailedResponses); + } + if (result.getNumberOfRowsPassed() > 0) { + result.setStatus(ApiStatus.PARTIAL_SUCCESS); + } else { + result.setStatus(ApiStatus.FAILURE); + } + } + return result; + }); + + BULK_JOBS.put(jobId, mergedJob); + + mergedJob.whenComplete( + (result, throwable) -> { + BULK_JOB_PERMITS.release(); + CompletableFuture.delayedExecutor(5, TimeUnit.MINUTES) + .execute(() -> BULK_JOBS.remove(jobId)); + }); + + return mergedJob; + } + + /** + * Fields used to hydrate inherited relationships for changed entities during bulk updates. + * + *

Use PUT-update fields as baseline and explicitly include inheritable fields so repositories + * with inheritance beyond owners/domains (for example retentionPeriod or reviewers) keep behavior + * intact without loading every allowed field. + */ + Fields getBulkUpdateInheritanceFields() { + Set bulkFields = new HashSet<>(r.putFields.getFieldList()); + String inheritableFields = r.getInheritableFields(); + if (!nullOrEmpty(inheritableFields)) { + for (String field : inheritableFields.split(",")) { + String normalized = field.trim(); + if (!normalized.isEmpty() && r.allowedFields.contains(normalized)) { + bulkFields.add(normalized); + } + } + } + return new Fields(r.allowedFields, bulkFields); + } + + /** + * Returns true when a connector-supplied entity is provably unchanged from what is stored, so + * the bulk update path can skip field hydration and per-field diffing. Requires a non-empty + * sourceHash on both the incoming entity and the stored original that match, an existing + * non-deleted original, and that the FQN appears only once in the batch (duplicate FQNs need + * the full updater path so each occurrence diffs against a fresh snapshot). + */ + boolean isSourceHashUnchanged( + T entity, Map hydratedOriginalByFqn, Map updateFrequencyByFqn) { + String fqn = entity.getFullyQualifiedName(); + if (nullOrEmpty(fqn) || updateFrequencyByFqn.getOrDefault(fqn, 0) > 1) { + return false; + } + String incomingHash = entity.getSourceHash(); + if (nullOrEmpty(incomingHash)) { + return false; + } + T original = hydratedOriginalByFqn.get(fqn); + if (original == null || Boolean.TRUE.equals(original.getDeleted())) { + return false; + } + return incomingHash.equals(original.getSourceHash()); + } + + void bulkUpdateEntities( + UriInfo uriInfo, + List updateEntities, + Map existingByFqn, + String userName, + boolean overrideMetadata, + List successRequests, + List failedRequests, + List entityLatenciesNanos) { + + if (updateEntities.isEmpty()) return; + + long batchStartTime = System.nanoTime(); + + // Batch load fields once per unique FQN. Duplicate rows in the same bulk request + // should reuse a hydrated original snapshot. + Map hydratedOriginalByFqn = new LinkedHashMap<>(); + Map updateFrequencyByFqn = new HashMap<>(); + for (T entity : updateEntities) { + String fqn = entity.getFullyQualifiedName(); + if (nullOrEmpty(fqn)) { + continue; + } + updateFrequencyByFqn.merge(fqn, 1, Integer::sum); + T original = existingByFqn.get(fqn); + if (original != null) { + hydratedOriginalByFqn.putIfAbsent(fqn, original); + } + } + + // sourceHash fast-path: skip entities whose connector-supplied sourceHash matches the + // stored value. This avoids field hydration and per-field diffing for unchanged entities. + // A skipped entity is reported as a no-change success - identical to the outcome of a full + // diff that finds nothing changed - so callers see no behavioral difference. Disabled when + // overrideMetadata is set: the caller explicitly wants stored metadata overwritten now, so a + // matching sourceHash must not short-circuit that. + List entitiesToProcess = new ArrayList<>(); + for (T entity : updateEntities) { + if (!overrideMetadata + && isSourceHashUnchanged(entity, hydratedOriginalByFqn, updateFrequencyByFqn)) { + successRequests.add( + new BulkResponse() + .withRequest(entity.getFullyQualifiedName()) + .withStatus(Status.OK.getStatusCode())); + entityLatenciesNanos.add(0L); + recordEntityMetrics(r.entityType, 0L, 0, true); + } else { + entitiesToProcess.add(entity); + } + } + if (entitiesToProcess.isEmpty()) return; + + // Hydrate only the originals of entities that still need a full diff. + Map originalsToHydrateByFqn = new LinkedHashMap<>(); + for (T entity : entitiesToProcess) { + T original = hydratedOriginalByFqn.get(entity.getFullyQualifiedName()); + if (original != null) { + originalsToHydrateByFqn.putIfAbsent(entity.getFullyQualifiedName(), original); + } + } + List originalsForHydration = new ArrayList<>(originalsToHydrateByFqn.values()); + try { + r.setFieldsInBulk(r.putFields, originalsForHydration); + } catch (Exception e) { + LOG.error("setFieldsInBulk failed, marking all updates as failed", e); + for (T entity : entitiesToProcess) { + failedRequests.add( + new BulkResponse() + .withRequest(entity.getFullyQualifiedName()) + .withStatus(Status.BAD_REQUEST.getStatusCode()) + .withMessage("Batch field loading failed: " + e.getMessage())); + } + return; + } + + // Per-entity updater (relationships + change description) + List.EntityUpdater> updaters = new ArrayList<>(); + + try (var ignored = phase("entityUpdaters")) { + for (T entity : entitiesToProcess) { + try { + String fqn = entity.getFullyQualifiedName(); + T hydratedOriginal = hydratedOriginalByFqn.get(fqn); + if (hydratedOriginal == null) { + failedRequests.add( + new BulkResponse() + .withRequest(fqn) + .withStatus(Status.BAD_REQUEST.getStatusCode()) + .withMessage("Entity does not exist")); + continue; + } + T original = + updateFrequencyByFqn.getOrDefault(fqn, 0) > 1 + ? JsonUtils.deepCopy(hydratedOriginal, r.entityClass) + : hydratedOriginal; + entity.setUpdatedBy(userName); + entity.setUpdatedAt(System.currentTimeMillis()); + + if (Boolean.TRUE.equals(original.getDeleted())) { + r.restoreEntity(entity.getUpdatedBy(), original.getId()); + } + + EntityRepository.EntityUpdater updater = + r.getUpdater(original, entity, EntityRepository.Operation.PUT, null); + updater.setOverrideMetadata(overrideMetadata); + updater.updateWithDeferredStore(); + updaters.add(updater); + } catch (Exception e) { + failedRequests.add( + new BulkResponse() + .withRequest(entity.getFullyQualifiedName()) + .withStatus(Status.BAD_REQUEST.getStatusCode()) + .withMessage(e.getMessage())); + } + } + } + + if (updaters.isEmpty()) return; + List.EntityUpdater> changedUpdaters = + updaters.stream() + .filter(updater -> updater.isVersionChanged() || updater.isEntityChanged()) + .toList(); + Fields bulkInheritanceFields = getBulkUpdateInheritanceFields(); + + // Batch DB writes + try { + try (var ignored = phase("batchDbWrites")) { + // Batch version history inserts + List historyIds = new ArrayList<>(); + List historyExtensions = new ArrayList<>(); + List historyJsons = new ArrayList<>(); + for (EntityRepository.EntityUpdater updater : changedUpdaters) { + if (updater.isVersionChanged()) { + historyIds.add(updater.getOriginal().getId()); + historyExtensions.add( + EntityUtil.getVersionExtension(r.entityType, updater.getOriginal().getVersion())); + historyJsons.add(JsonUtils.pojoToJson(updater.getOriginal())); + } + } + if (!historyIds.isEmpty()) { + r.daoCollection + .entityExtensionDAO() + .insertMany(historyIds, historyExtensions, r.entityType, historyJsons); + } + + // Batch entity row updates + List entitiesToStore = new ArrayList<>(); + for (EntityRepository.EntityUpdater updater : changedUpdaters) { + entitiesToStore.add(updater.getUpdated()); + } + if (!entitiesToStore.isEmpty()) { + r.updateMany(entitiesToStore); + } + } + + List changedEntities = + changedUpdaters.stream().map(EntityRepository.EntityUpdater::getUpdated).toList(); + if (!changedEntities.isEmpty()) { + try (var ignored = phase("setInheritedFields")) { + // Only changed entities need inheritance hydration for downstream side effects. + r.setInheritedFields(changedEntities, bulkInheritanceFields); + } + try (var ignored = phase("invalidateCacheBulk")) { + r.invalidateMany(changedEntities); + } + try (var ignored = phase("postUpdateEvents")) { + List changeEventJsons = new ArrayList<>(); + for (var updater : changedUpdaters) { + r.postUpdate(updater.getOriginal(), updater.getUpdated()); + updater.runDeferredReactOperations(); + var changeType = updater.incrementalFieldsChanged() ? ENTITY_UPDATED : ENTITY_NO_CHANGE; + buildChangeEventJsonForBulkOperation(updater.getUpdated(), changeType, userName) + .ifPresent(changeEventJsons::add); + } + insertChangeEventsBatch(changeEventJsons); + } + } + + // Per-entity success + metrics (includes no-change updates). + long batchDuration = System.nanoTime() - batchStartTime; + long perEntityDuration = batchDuration / updaters.size(); + for (var updater : updaters) { + entityLatenciesNanos.add(perEntityDuration); + recordEntityMetrics(r.entityType, perEntityDuration, 0, true); + successRequests.add( + new BulkResponse() + .withRequest(updater.getUpdated().getFullyQualifiedName()) + .withStatus(Status.OK.getStatusCode())); + } + } catch (Exception batchError) { + LOG.warn("Batch update store failed, falling back to per-entity updates", batchError); + List.EntityUpdater> succeededUpdaters = new ArrayList<>(); + for (var updater : updaters) { + try { + if (updater.isVersionChanged() || updater.isEntityChanged()) { + updater.storeUpdate(); + r.invalidate(updater.getUpdated()); + } + succeededUpdaters.add(updater); + } catch (Exception e) { + failedRequests.add( + new BulkResponse() + .withRequest(updater.getUpdated().getFullyQualifiedName()) + .withStatus(Status.BAD_REQUEST.getStatusCode()) + .withMessage(e.getMessage())); + } + } + if (!succeededUpdaters.isEmpty()) { + List fallbackChanged = + succeededUpdaters.stream() + .filter(updater -> updater.isVersionChanged() || updater.isEntityChanged()) + .map(EntityRepository.EntityUpdater::getUpdated) + .toList(); + if (!fallbackChanged.isEmpty()) { + try (var ignored = phase("invalidateCacheBulk")) { + r.invalidateMany(fallbackChanged); + } + r.setInheritedFields(fallbackChanged, bulkInheritanceFields); + } + + List changeEventJsons = new ArrayList<>(); + for (var updater : succeededUpdaters) { + if (updater.isVersionChanged() || updater.isEntityChanged()) { + r.postUpdate(updater.getOriginal(), updater.getUpdated()); + updater.runDeferredReactOperations(); + var changeType = updater.incrementalFieldsChanged() ? ENTITY_UPDATED : ENTITY_NO_CHANGE; + buildChangeEventJsonForBulkOperation(updater.getUpdated(), changeType, userName) + .ifPresent(changeEventJsons::add); + } + successRequests.add( + new BulkResponse() + .withRequest(updater.getUpdated().getFullyQualifiedName()) + .withStatus(Status.OK.getStatusCode())); + } + insertChangeEventsBatch(changeEventJsons); + } + } + } + + BulkOperationResult bulkCreateOrUpdateEntitiesSequential( + UriInfo uriInfo, + List entities, + String userName, + Map existingByFqn, + boolean overrideMetadata) { + + BulkOperationResult result = new BulkOperationResult(); + result.setStatus(ApiStatus.SUCCESS); + + List successRequests = new ArrayList<>(); + List failedRequests = new ArrayList<>(); + + long bulkStartTime = System.nanoTime(); + List entityLatenciesNanos = new ArrayList<>(); + + // Separate into creates and updates using the pre-fetched map + // For duplicate FQNs within the batch, first occurrence goes to creates, + // subsequent occurrences go to updates (processed after creates) + List newEntities = new ArrayList<>(); + List updateEntities = new ArrayList<>(); + Set seenNewFqns = new HashSet<>(); + for (T entity : entities) { + String fqn = entity.getFullyQualifiedName(); + if (existingByFqn.containsKey(fqn)) { + updateEntities.add(entity); + } else if (seenNewFqns.contains(fqn)) { + updateEntities.add(entity); + } else { + seenNewFqns.add(fqn); + newEntities.add(entity); + } + } + + // Batch create new entities + if (!newEntities.isEmpty()) { + long batchStartTime = System.nanoTime(); + try { + r.createManyEntities(newEntities); + long batchDuration = System.nanoTime() - batchStartTime; + long perEntityDuration = batchDuration / newEntities.size(); + List createdChangeEventJsons = new ArrayList<>(); + for (T entity : newEntities) { + entityLatenciesNanos.add(perEntityDuration); + recordEntityMetrics(r.entityType, perEntityDuration, 0, true); + successRequests.add( + new BulkResponse() + .withRequest(entity.getFullyQualifiedName()) + .withStatus(Status.OK.getStatusCode())); + buildChangeEventJsonForBulkOperation(entity, ENTITY_CREATED, userName) + .ifPresent(createdChangeEventJsons::add); + } + insertChangeEventsBatch(createdChangeEventJsons); + } catch (Exception batchError) { + LOG.warn("Batch create failed, falling back to per-entity creates", batchError); + for (T entity : newEntities) { + long entityStartTime = System.nanoTime(); + try { + PutResponse putResponse = r.createOrUpdate(uriInfo, entity, userName); + long entityDuration = System.nanoTime() - entityStartTime; + entityLatenciesNanos.add(entityDuration); + recordEntityMetrics(r.entityType, entityDuration, 0, true); + successRequests.add( + new BulkResponse() + .withRequest(entity.getFullyQualifiedName()) + .withStatus(Status.OK.getStatusCode())); + createChangeEventForBulkOperation( + putResponse.getEntity(), putResponse.getChangeType(), userName); + } catch (Exception e) { + long entityDuration = System.nanoTime() - entityStartTime; + entityLatenciesNanos.add(entityDuration); + if (isDuplicateKeyException(e)) { + LOG.debug( + "Entity already exists (duplicate key), treating as success: {}", + entity.getFullyQualifiedName()); + recordEntityMetrics(r.entityType, entityDuration, 0, true); + successRequests.add( + new BulkResponse() + .withRequest(entity.getFullyQualifiedName()) + .withStatus(Status.OK.getStatusCode())); + } else { + recordEntityMetrics(r.entityType, entityDuration, 0, false); + failedRequests.add( + new BulkResponse() + .withRequest(entity.getFullyQualifiedName()) + .withStatus(Status.BAD_REQUEST.getStatusCode()) + .withMessage(e.getMessage())); + } + } + } + } + } + + // For duplicate FQNs within the batch, refresh existingByFqn with newly created entities + if (!updateEntities.isEmpty()) { + List updateFqns = + updateEntities.stream() + .map(T::getFullyQualifiedName) + .filter(fqn -> !existingByFqn.containsKey(fqn)) + .distinct() + .collect(Collectors.toList()); + if (!updateFqns.isEmpty()) { + List newlyCreated = r.dao.findEntityByNames(updateFqns, Include.ALL); + for (T created : newlyCreated) { + existingByFqn.put(created.getFullyQualifiedName(), created); + } + } + + // Filter out entities whose original doesn't exist (e.g., duplicate FQN whose + // first occurrence failed to create). These can't be updated — report as failed. + Iterator it = updateEntities.iterator(); + while (it.hasNext()) { + T entity = it.next(); + if (!existingByFqn.containsKey(entity.getFullyQualifiedName())) { + it.remove(); + failedRequests.add( + new BulkResponse() + .withRequest(entity.getFullyQualifiedName()) + .withStatus(Status.BAD_REQUEST.getStatusCode()) + .withMessage("Entity does not exist and could not be created")); + } + } + } + + // Batch update existing entities + bulkUpdateEntities( + uriInfo, + updateEntities, + existingByFqn, + userName, + overrideMetadata, + successRequests, + failedRequests, + entityLatenciesNanos); + + long totalDurationNanos = System.nanoTime() - bulkStartTime; + + result.setNumberOfRowsProcessed(entities.size()); + result.setNumberOfRowsPassed(successRequests.size()); + result.setNumberOfRowsFailed(failedRequests.size()); + result.setSuccessRequest(successRequests); + result.setFailedRequest(failedRequests); + + if (!failedRequests.isEmpty()) { + result.setStatus(successRequests.isEmpty() ? ApiStatus.FAILURE : ApiStatus.PARTIAL_SUCCESS); + } + + // Calculate metrics + long avgEntityLatencyMs = 0; + long maxEntityLatencyMs = 0; + if (!entityLatenciesNanos.isEmpty()) { + avgEntityLatencyMs = + entityLatenciesNanos.stream().mapToLong(Long::longValue).sum() + / entityLatenciesNanos.size() + / 1_000_000; + maxEntityLatencyMs = + entityLatenciesNanos.stream().mapToLong(Long::longValue).max().orElse(0) / 1_000_000; + } + + recordBulkMetrics( + r.entityType, + entities.size(), + successRequests.size(), + totalDurationNanos, + avgEntityLatencyMs, + maxEntityLatencyMs); + + LOG.info( + "Bulk operation completed: {} succeeded, {} failed out of {} total, took {}ms", + successRequests.size(), + failedRequests.size(), + entities.size(), + totalDurationNanos / 1_000_000); + + return result; + } + + Optional getBulkJobStatus(String jobId) { + CompletableFuture job = BULK_JOBS.get(jobId); + if (job == null) { + return Optional.empty(); + } + + if (job.isDone() && !job.isCompletedExceptionally()) { + try { + return Optional.of(job.get()); + } catch (ExecutionException | InterruptedException e) { + LOG.error("Error retrieving job status for jobId: {}", jobId, e); + java.lang.Thread.currentThread().interrupt(); + return Optional.empty(); + } + } + + BulkOperationResult inProgress = new BulkOperationResult(); + inProgress.setStatus(ApiStatus.RUNNING); + return Optional.of(inProgress); + } + + @Transaction + PutResponse createOrUpdateWithOriginal( + UriInfo uriInfo, T updated, T original, String updatedBy) { + if (r.lockManager != null) { + r.lockManager.checkModificationAllowed(updated); + } + if (original == null) { + return new PutResponse<>( + Status.CREATED, r.withHref(uriInfo, r.createNewEntity(updated)), ENTITY_CREATED); + } + return r.update(uriInfo, original, updated, updatedBy, null); + } + + void createChangeEventForBulkOperation(T entity, EventType eventType, String userName) { + Optional changeEventJson = + buildChangeEventJsonForBulkOperation(entity, eventType, userName); + if (changeEventJson.isEmpty()) { + return; + } + try { + Entity.getCollectionDAO().changeEventDAO().insert(changeEventJson.get()); + } catch (Exception e) { + LOG.error("Failed to create change event for bulk operation", e); + } + } + + Optional buildChangeEventJsonForBulkOperation( + T entity, EventType eventType, String userName) { + try { + if (eventType.equals(ENTITY_NO_CHANGE)) { + return Optional.empty(); + } + + ChangeEvent changeEvent = + FormatterUtil.createChangeEventForEntity(userName, eventType, entity); + + if (changeEvent.getEntity() != null) { + Object entityObject = changeEvent.getEntity(); + changeEvent = copyChangeEvent(changeEvent); + changeEvent.setEntity(JsonUtils.pojoToMaskedJson(entityObject)); + } + + LOG.debug( + "Recording change event for bulk operation {}:{}:{}:{}", + changeEvent.getTimestamp(), + changeEvent.getEntityId(), + changeEvent.getEventType(), + changeEvent.getEntityType()); + + return Optional.of(JsonUtils.pojoToJson(changeEvent)); + } catch (Exception e) { + LOG.error("Failed to create change event for bulk operation", e); + return Optional.empty(); + } + } + + void insertChangeEventsBatch(List changeEvents) { + if (changeEvents == null || changeEvents.isEmpty()) { + return; + } + try { + Entity.getCollectionDAO().changeEventDAO().insertBatch(changeEvents); + } catch (Exception batchError) { + LOG.error("Failed to insert change events batch", batchError); + } + } + + static ChangeEvent copyChangeEvent(ChangeEvent changeEvent) { + return new ChangeEvent() + .withId(changeEvent.getId()) + .withEventType(changeEvent.getEventType()) + .withEntityId(changeEvent.getEntityId()) + .withEntityType(changeEvent.getEntityType()) + .withUserName(changeEvent.getUserName()) + .withImpersonatedBy(changeEvent.getImpersonatedBy()) + .withTimestamp(changeEvent.getTimestamp()) + .withChangeDescription(changeEvent.getChangeDescription()) + .withCurrentVersion(changeEvent.getCurrentVersion()) + .withPreviousVersion(changeEvent.getPreviousVersion()) + .withEntityFullyQualifiedName(changeEvent.getEntityFullyQualifiedName()); + } + + BulkOperationResult bulkCreateOrUpdateEntities( + UriInfo uriInfo, List entities, String userName, Map existingByFqn) { + return bulkCreateOrUpdateEntities(uriInfo, entities, userName, existingByFqn, false); + } + + BulkOperationResult bulkCreateOrUpdateEntities( + UriInfo uriInfo, + List entities, + String userName, + Map existingByFqn, + boolean overrideMetadata) { + return bulkCreateOrUpdateEntitiesSequential( + uriInfo, entities, userName, existingByFqn, overrideMetadata); + } + + /** + * Deletes entities of this type within {@code request.scopeFqn} that the ingestion connector did + * not report in the current run. The connector sends the set of FQNs it saw ({@code + * request.seenFqns}); any live entity under the scope whose FQN is not in that set is considered + * stale. By default the deletion is soft; pass {@code hardDelete=true} on the request to + * hard-delete the stale entities. + * + *

FQNs are compared by hash so quoting or case differences between the connector-supplied and + * stored values never cause spurious deletes. An empty scope yields zero deletions - it is never + * interpreted as "everything is stale". Each delete runs in its own transaction so a single + * failure does not roll back the rest of the batch. + */ + BulkOperationResult bulkDeleteStaleEntities(BulkDeleteStaleRequest request, String deletedBy) { + validateScopeRequest(request.getScopeEntityType(), request.getScopeFqn()); + if (nullOrEmpty(request.getSeenFqns())) { + // An empty seen-set cannot be distinguished from a connector run that crashed or discovered + // nothing, so it must never be interpreted as "every entity under the scope is stale" - that + // would silently delete the whole service/database. Treat it as zero deletions, mirroring the + // scope-not-found path. + LOG.warn( + "deleteStale for scope {} '{}' received an empty seenFqns; treating as zero deletions " + + "rather than marking the entire scope stale", + request.getScopeEntityType(), + request.getScopeFqn()); + return buildStaleDeletionResult( + Boolean.TRUE.equals(request.getDryRun()), new ArrayList<>(), new ArrayList<>()); + } + if (!scopeExists(request.getScopeEntityType(), request.getScopeFqn())) { + LOG.warn( + "deleteStale scope {} '{}' not found; nothing to delete this run", + request.getScopeEntityType(), + request.getScopeFqn()); + return buildStaleDeletionResult( + Boolean.TRUE.equals(request.getDryRun()), new ArrayList<>(), new ArrayList<>()); + } + boolean dryRun = Boolean.TRUE.equals(request.getDryRun()); + boolean hardDelete = Boolean.TRUE.equals(request.getHardDelete()); + boolean recursive = !Boolean.FALSE.equals(request.getRecursive()); + List successRequests = new ArrayList<>(); + List failedRequests = new ArrayList<>(); + Set deletedHashes = new HashSet<>(); + for (EntityDAO.EntityIdFqnPair stale : + findStaleEntities(request.getScopeFqn(), request.getSeenFqns())) { + String fqnHash = FullyQualifiedName.buildHash(stale.fqn); + if (dryRun || isCoveredByDeletedAncestor(fqnHash, deletedHashes)) { + successRequests.add(staleSuccess(stale.fqn)); + continue; + } + try { + r.deleteInternal(deletedBy, stale.id, recursive, hardDelete); + deletedHashes.add(fqnHash); + successRequests.add(staleSuccess(stale.fqn)); + } catch (Exception e) { + LOG.warn("Failed to delete stale {} '{}': {}", r.entityType, stale.fqn, e.getMessage()); + failedRequests.add( + new BulkResponse() + .withRequest(stale.fqn) + .withStatus(Status.INTERNAL_SERVER_ERROR.getStatusCode()) + .withMessage(e.getMessage())); + } + } + return buildStaleDeletionResult(dryRun, successRequests, failedRequests); + } + + /** + * Rejects a malformed request before any work runs: {@code scopeFqn} and {@code scopeEntityType} + * must be present and {@code scopeEntityType} must resolve to a registered entity type. These are + * caller mistakes (typos, swapped fields) and fail with 400. An empty {@code seenFqns} is handled + * separately in {@link #bulkDeleteStaleEntities} as a safe zero-deletion no-op rather than a 400, + * since it is a well-formed request whose intent is genuinely ambiguous. + */ + void validateScopeRequest(String scopeEntityType, String scopeFqn) { + if (nullOrEmpty(scopeFqn)) { + throw BadRequestException.of("scopeFqn is required for deleteStale"); + } + if (nullOrEmpty(scopeEntityType)) { + throw BadRequestException.of("scopeEntityType is required for deleteStale"); + } + if (!Entity.hasEntityRepository(resolveScopeEntityType(scopeEntityType))) { + throw BadRequestException.of( + String.format("Unsupported scopeEntityType '%s' for deleteStale", scopeEntityType)); + } + } + + /** + * Returns whether a live entity with FQN {@code scopeFqn} exists under the resolved scope type. A + * missing scope is not an error - it means the connector has nothing to reconcile yet (scope not + * persisted) or the scope was already removed - so the caller treats it as zero deletions. + */ + boolean scopeExists(String scopeEntityType, String scopeFqn) { + EntityRepository scopeRepository = + Entity.getEntityRepository(resolveScopeEntityType(scopeEntityType)); + return scopeRepository.findByNameOrNull(scopeFqn, Include.NON_DELETED) != null; + } + + /** + * Resolves a generic {@code service} scope to the concrete service type that owns this entity + * type (for example {@code pipeline} -> {@code pipelineService}), mirroring how {@link + * org.openmetadata.service.jdbi3.ListFilter} interprets the {@code service} query param. + * Connectors scope service-level stale deletion with the generic {@code service} key, so without + * this the request would be rejected as an unknown entity type. Any concrete scope type is + * returned unchanged. + */ + String resolveScopeEntityType(String scopeEntityType) { + return Entity.FIELD_SERVICE.equals(scopeEntityType) + ? Entity.getServiceType(r.entityType) + : scopeEntityType; + } + + /** + * Returns the live entities under {@code scopeFqn} that are not present in {@code seenFqns}, + * sorted shallowest-FQN-first so a recursive delete of an ancestor is processed before its + * descendants. Comparison is by FQN hash to be quoting and case insensitive. + */ + List findStaleEntities(String scopeFqn, List seenFqns) { + List scopeEntities = + r.dao.listDescendantIdFqnByPrefixNonDeleted(scopeFqn); + if (scopeEntities.isEmpty()) { + return List.of(); + } + Set seenHashes = new HashSet<>(); + for (String seenFqn : listOrEmpty(seenFqns)) { + try { + seenHashes.add(FullyQualifiedName.buildHash(seenFqn)); + } catch (Exception e) { + LOG.warn("Ignoring malformed seen FQN '{}' in stale deletion request", seenFqn); + } + } + return scopeEntities.stream() + .filter(pair -> !seenHashes.contains(FullyQualifiedName.buildHash(pair.fqn))) + .sorted(Comparator.comparingInt(pair -> FullyQualifiedName.split(pair.fqn).length)) + .toList(); + } + + boolean isCoveredByDeletedAncestor(String fqnHash, Set deletedHashes) { + if (deletedHashes.isEmpty()) { + return false; + } + int separator = fqnHash.lastIndexOf('.'); + while (separator > 0) { + String ancestor = fqnHash.substring(0, separator); + if (deletedHashes.contains(ancestor)) { + return true; + } + separator = ancestor.lastIndexOf('.'); + } + return false; + } + + BulkResponse staleSuccess(String fqn) { + return new BulkResponse().withRequest(fqn).withStatus(Status.OK.getStatusCode()); + } + + BulkOperationResult buildStaleDeletionResult( + boolean dryRun, List successRequests, List failedRequests) { + BulkOperationResult result = new BulkOperationResult(); + result.setDryRun(dryRun); + result.setStatus(failedRequests.isEmpty() ? ApiStatus.SUCCESS : ApiStatus.PARTIAL_SUCCESS); + result.setNumberOfRowsProcessed(successRequests.size() + failedRequests.size()); + result.setNumberOfRowsPassed(successRequests.size()); + result.setNumberOfRowsFailed(failedRequests.size()); + result.setSuccessRequest(successRequests); + result.setFailedRequest(failedRequests); + return result; + } + + static boolean isDuplicateKeyException(Exception e) { + Throwable cause = e.getCause(); + if (cause instanceof SQLException sqlEx) { + // MySQL: error code 1062 = ER_DUP_ENTRY + // PostgreSQL: SQL state "23505" = unique_violation + return sqlEx.getErrorCode() == 1062 || "23505".equals(sqlEx.getSQLState()); + } + return false; + } + + void recordEntityMetrics( + String entityType, long durationNanos, long queueWaitNanos, boolean success) { + // Per-entity processing time (cached, no histogram to reduce Prometheus cardinality) + // This fires for EVERY entity in a bulk operation, so we use simple timers. + // The bulk.operation.latency metric has histograms for percentile analysis. + String latencyKey = entityType + "|" + success; + Timer latencyTimer = + ENTITY_LATENCY_TIMERS.computeIfAbsent( + latencyKey, + k -> + Timer.builder("bulk.entity.latency") + .tag("entity", entityType) + .tag("success", String.valueOf(success)) + .register(Metrics.globalRegistry)); + latencyTimer.record(durationNanos, TimeUnit.NANOSECONDS); + + // Queue wait time (cached, simple timer) + Timer queueTimer = + ENTITY_QUEUE_WAIT_TIMERS.computeIfAbsent( + entityType, + k -> + Timer.builder("bulk.entity.queue_wait") + .tag("entity", entityType) + .register(Metrics.globalRegistry)); + queueTimer.record(queueWaitNanos, TimeUnit.NANOSECONDS); + } + + void recordBulkMetrics( + String entityType, + int totalEntities, + int successCount, + long durationNanos, + long avgEntityMs, + long maxEntityMs) { + // Total bulk operation time (cached) + Timer operationTimer = + BULK_OPERATION_TIMERS.computeIfAbsent( + entityType, + k -> + Timer.builder("bulk.operation.latency") + .tag("entity", entityType) + .publishPercentileHistogram(true) + .register(Metrics.globalRegistry)); + operationTimer.record(durationNanos, TimeUnit.NANOSECONDS); + + // Batch size distribution (cached) + DistributionSummary batchSizeSummary = + BATCH_SIZE_SUMMARIES.computeIfAbsent( + entityType, + k -> + DistributionSummary.builder("bulk.operation.batch_size") + .tag("entity", entityType) + .register(Metrics.globalRegistry)); + batchSizeSummary.record(totalEntities); + + // Success rate as distribution (cached, avoids gauge memory leak) + if (totalEntities > 0) { + DistributionSummary successRateSummary = + SUCCESS_RATE_SUMMARIES.computeIfAbsent( + entityType, + k -> + DistributionSummary.builder("bulk.operation.success_rate") + .tag("entity", entityType) + .register(Metrics.globalRegistry)); + successRateSummary.record(successCount * 100.0 / totalEntities); + } + + // Record success and failure counts for alerting (Micrometer caches counters internally) + Metrics.counter("bulk.operation.entities.success", "entity", entityType) + .increment(successCount); + Metrics.counter("bulk.operation.entities.failed", "entity", entityType) + .increment(totalEntities - successCount); + } + + void handleBulkOperationError(T entity, Exception e, List failedRequests) { + String fqn = entity.getFullyQualifiedName(); + int statusCode; + String message; + + // Categorize errors properly + if (e instanceof jakarta.ws.rs.WebApplicationException wae) { + statusCode = wae.getResponse().getStatus(); + message = e.getMessage(); + LOG.warn("Entity {} failed with status {}: {}", fqn, statusCode, message); + } else if (e instanceof java.sql.SQLException) { + statusCode = Status.INTERNAL_SERVER_ERROR.getStatusCode(); + message = "Database error: " + e.getMessage(); + LOG.error("Database error processing entity {}", fqn, e); + } else if (e instanceof IllegalArgumentException || e instanceof IllegalStateException) { + statusCode = Status.BAD_REQUEST.getStatusCode(); + message = e.getMessage(); + LOG.warn("Validation error for entity {}: {}", fqn, message); + } else { + statusCode = Status.INTERNAL_SERVER_ERROR.getStatusCode(); + message = "Unexpected error: " + e.getMessage(); + LOG.error("Unexpected error processing entity {}", fqn, e); + } + + failedRequests.add( + new BulkResponse().withRequest(fqn).withStatus(statusCode).withMessage(message)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java index bf59d53372ed..c103d760db38 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java @@ -44,7 +44,7 @@ import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.tags.ClassificationResource; import org.openmetadata.service.security.policyevaluator.PolicyConditionUpdater; @@ -350,13 +350,13 @@ public void updateName(Classification updated) { // Capture the descendants so the post-write pass can re-evict any entry a racing reader // re-populated with the pre-rename row between this call and tagDAO.updateFqn below. The // pass below runs after updateFqn but inside this transaction — see - // EntityRepository.invalidateCacheForRenameCascade for the residual pre-commit window. + // EntityCacheInvalidator.invalidateCacheForRenameCascade for the residual pre-commit window. List renamedTags = - invalidateCacheForRenameCascade(Entity.TAG, oldFqn); + EntityCacheInvalidator.invalidateCacheForRenameCascade(Entity.TAG, oldFqn); // Drop cached entity JSON / bundle for every entity tagged with any tag under this // classification. Tags live in the TAG entity table with FQNs starting with the // classification FQN, so the descendant helper finds them correctly. - invalidateCacheForTaggedEntitiesAndDescendants(Entity.TAG, oldFqn); + EntityCacheInvalidator.invalidateCacheForTaggedEntitiesAndDescendants(Entity.TAG, oldFqn); daoCollection.tagDAO().updateFqn(oldFqn, newFqn); daoCollection .tagUsageDAO() @@ -372,7 +372,7 @@ public void updateName(Classification updated) { condition, oldFqn, newFqn, PolicyConditionUpdater.TAG_FUNCTIONS)); invalidateClassification(updated.getId()); - finishInvalidateCacheForRenameCascade(Entity.TAG, renamedTags); + EntityCacheInvalidator.finishInvalidateCacheForRenameCascade(Entity.TAG, renamedTags); } private void updateEntityLinks(String oldFqn, String newFqn, Classification updated) { @@ -393,7 +393,7 @@ private void updateEntityLinks(String oldFqn, String newFqn, Classification upda private void invalidateClassification(UUID classificationId) { // Name of the classification changed. Invalidate the classification and all the children tags - CACHE_WITH_ID.invalidate(new ImmutablePair<>(CLASSIFICATION, classificationId)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(CLASSIFICATION, classificationId)); List tagRecords = findToRecords(classificationId, CLASSIFICATION, Relationship.CONTAINS, TAG); for (EntityRelationshipRecord tagRecord : tagRecords) { @@ -405,7 +405,7 @@ private void invalidateTags(UUID tagId) { // The name of the tag changed. Invalidate that tag and all the children from the cache List tagRecords = findToRecords(tagId, TAG, Relationship.CONTAINS, TAG); - CACHE_WITH_ID.invalidate(new ImmutablePair<>(TAG, tagId)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(TAG, tagId)); for (EntityRelationshipRecord tagRecord : tagRecords) { invalidateTags(tagRecord.getId()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java new file mode 100644 index 000000000000..fe6acd2a7b38 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java @@ -0,0 +1,1187 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.schema.type.Relationship.CONTAINS; +import static org.openmetadata.service.jdbi3.ListFilter.escapeApostrophe; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.tuple.Pair; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.jdbi.v3.sqlobject.statement.UseRowMapper; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.entity.classification.Classification; +import org.openmetadata.schema.entity.classification.Tag; +import org.openmetadata.schema.entity.data.GlossaryTerm; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.type.TagLabelMetadata; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.ClassificationTagDAOs.TagUsageDAO.TagLabelMapper; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlBatch; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.resources.tags.TagLabelUtil; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.jdbi.BindConcat; +import org.openmetadata.service.util.jdbi.BindFQN; +import org.openmetadata.service.util.jdbi.BindListFQN; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public interface ClassificationTagDAOs { + @CreateSqlObject + ClassificationDAO classificationDAO(); + + @CreateSqlObject + TagDAO tagDAO(); + + @CreateSqlObject + TagUsageDAO tagUsageDAO(); + + interface ClassificationDAO extends EntityDAO { + @Override + default String getTableName() { + return "classification"; + } + + @Override + default Class getEntityClass() { + return Classification.class; + } + + // Much more efficient: Use IN clause with proper index usage + @SqlQuery( + "SELECT classificationHash, COUNT(*) AS termCount " + + "FROM tag " + + "WHERE classificationHash IN () " + + "AND deleted = FALSE " + + "GROUP BY classificationHash") + @RegisterRowMapper(TermCountMapper.class) + List> bulkGetTermCounts(@BindList("hashArray") List hashArray); + + class TermCountMapper implements RowMapper> { + @Override + public Pair map(ResultSet rs, StatementContext ctx) throws SQLException { + return Pair.of(rs.getString("classificationHash"), rs.getInt("termCount")); + } + } + } + + interface TagDAO extends EntityDAO { + @Override + default String getTableName() { + return "tag"; + } + + @Override + default Class getEntityClass() { + return Tag.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + private Pair buildTagQueryConditions(ListFilter filter) { + String parent = filter.getQueryParam("parent"); + boolean disabled = Boolean.parseBoolean(filter.getQueryParam("classification.disabled")); + + String baseJoin = + String.format( + "INNER JOIN entity_relationship er ON tag.id=er.toId AND er.relation=%s AND er.fromEntity='%s' " + + "INNER JOIN classification c ON er.fromId=c.id", + CONTAINS.ordinal(), Entity.CLASSIFICATION); + + StringBuilder mySqlCondition = new StringBuilder(baseJoin); + StringBuilder postgresCondition = new StringBuilder(baseJoin); + + if (parent != null) { + String parentFqnHash = FullyQualifiedName.buildHash(parent); + filter.queryParams.put("parentFqnPrefix", parentFqnHash + ".%"); + mySqlCondition.append(" AND tag.fqnHash LIKE :parentFqnPrefix"); + postgresCondition.append(" AND tag.fqnHash LIKE :parentFqnPrefix"); + } + + if (disabled) { + mySqlCondition.append( + " AND (JSON_EXTRACT(c.json, '$.disabled') = TRUE OR JSON_EXTRACT(tag.json, '$.disabled') = TRUE)"); + postgresCondition.append( + " AND (COALESCE((c.json->>'disabled')::boolean, FALSE) = TRUE OR COALESCE((tag.json->>'disabled')::boolean, FALSE) = TRUE)"); + } else if (filter.getQueryParam("classification.disabled") != null) { + mySqlCondition.append( + " AND (JSON_EXTRACT(c.json, '$.disabled') IS NULL OR JSON_EXTRACT(c.json, '$.disabled') = FALSE) AND (JSON_EXTRACT(tag.json, '$.disabled') IS NULL OR JSON_EXTRACT(tag.json, '$.disabled') = FALSE)"); + postgresCondition.append( + " AND COALESCE((c.json->>'disabled')::boolean, FALSE) = FALSE AND COALESCE((tag.json->>'disabled')::boolean, FALSE) = FALSE"); + } + + String tagCondition = filter.getCondition("tag"); + if (!tagCondition.isEmpty()) { + mySqlCondition.append(" ").append(tagCondition); + postgresCondition.append(" ").append(tagCondition); + } + + return Pair.of(mySqlCondition.toString(), postgresCondition.toString()); + } + + @Override + default int listCount(ListFilter filter) { + Pair conditions = buildTagQueryConditions(filter); + return listCount( + getTableName(), + getNameHashColumn(), + filter.getQueryParams(), + conditions.getLeft(), + conditions.getRight()); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + Pair conditions = buildTagQueryConditions(filter); + return listBefore( + getTableName(), + filter.getQueryParams(), + conditions.getLeft(), + conditions.getRight(), + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + Pair conditions = buildTagQueryConditions(filter); + return listAfter( + getTableName(), + filter.getQueryParams(), + conditions.getLeft(), + conditions.getRight(), + limit, + afterName, + afterId); + } + + @SqlQuery("select json FROM tag where fqnhash LIKE :concatFqnhash") + List getTagsStartingWithPrefix( + @BindConcat( + value = "concatFqnhash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE tag SET json = JSON_SET(json, '$.recognizers', CAST(:recognizers AS JSON)) " + + "WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') = :tagFqn", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE tag SET json = json::jsonb || json_build_object(" + + "'recognizers', :recognizers::jsonb " + + ")::jsonb WHERE json->>'fullyQualifiedName' = :tagFqn", + connectionType = POSTGRES) + void patchRecognizers(@Bind("tagFqn") String tagFqn, @Bind("recognizers") String recognizers); + } + + @RegisterRowMapper(TagLabelMapper.class) + interface TagUsageDAO { + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedBy, metadata) VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :reason, :appliedBy, :metadata)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedBy, metadata) VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :reason, :appliedBy, :metadata :: jsonb) ON CONFLICT (source, tagFQNHash, targetFQNHash) DO NOTHING", + connectionType = POSTGRES) + void applyTag( + @Bind("source") int source, + @Bind("tagFQN") String tagFQN, + @BindFQN("tagFQNHash") String tagFQNHash, + @BindFQN("targetFQNHash") String targetFQNHash, + @Bind("labelType") int labelType, + @Bind("state") int state, + @Bind("reason") String reason, + @Bind("appliedBy") String appliedBy, + @Bind("metadata") String metadata); + + default void applyTag( + int source, + String tagFQN, + String tagFQNHash, + String targetFQNHash, + int labelType, + int state, + String reason, + String appliedBy, + TagLabelMetadata metadata) { + this.applyTag( + source, + tagFQN, + tagFQNHash, + targetFQNHash, + labelType, + state, + reason, + appliedBy, + JsonUtils.pojoToJson(metadata)); + } + + default void applyTag( + int source, + String tagFQN, + String tagFQNHash, + String targetFQNHash, + int labelType, + int state, + String reason, + String appliedBy) { + this.applyTag( + source, + tagFQN, + tagFQNHash, + targetFQNHash, + labelType, + state, + reason, + appliedBy, + (String) null); + } + + default List getTags(String targetFQN) { + List tags = getTagsInternal(targetFQN); + tags.forEach(TagLabelUtil::applyTagCommonFieldsGracefully); + return tags; + } + + default Map> getTagsByPrefix( + String targetFQNPrefix, String postfix, boolean requiresFqnHash) { + String targetFQNPrefixHash = + requiresFqnHash ? FullyQualifiedName.buildHash(targetFQNPrefix) : targetFQNPrefix; + Map> resultSet = new LinkedHashMap<>(); + List> tags = getTagsInternalByPrefix(targetFQNPrefixHash, postfix); + tags.forEach( + pair -> { + String targetHash = pair.getLeft(); + TagLabel tagLabel = pair.getRight(); + List listOfTarget = new ArrayList<>(); + if (resultSet.containsKey(targetHash)) { + listOfTarget = resultSet.get(targetHash); + listOfTarget.add(tagLabel); + } else { + listOfTarget.add(tagLabel); + } + resultSet.put(targetHash, listOfTarget); + }); + return resultSet; + } + + @SqlQuery( + "SELECT source, tagFQN, labelType, state, reason, appliedAt, appliedBy, metadata FROM tag_usage WHERE targetFQNHash = :targetFQNHash ORDER BY tagFQN") + List getTagsInternal(@BindFQN("targetFQNHash") String targetFQNHash); + + @SqlQuery( + "SELECT targetFQNHash, source, tagFQN, labelType, state, reason, appliedAt, appliedBy, metadata " + + "FROM tag_usage " + + "WHERE targetFQNHash IN () " + + "ORDER BY targetFQNHash, tagFQN") + @UseRowMapper(TagLabelWithFQNHashMapper.class) + List getTagsInternalBatch( + @BindListFQN("targetFQNHashes") List targetFQNHashes); + + @SqlQuery( + "SELECT targetFQNHash, source, tagFQN, labelType, state, reason, appliedAt, appliedBy, metadata " + + "FROM tag_usage " + + "WHERE source = :source " + + "AND targetFQNHash IN () " + + "AND tagFQNHash LIKE :tagFQNHashPrefix " + + "ORDER BY targetFQNHash, tagFQN") + @UseRowMapper(TagLabelWithFQNHashMapper.class) + List getCertTagsInternalBatch( + @Bind("source") int source, + @BindListFQN("targetFQNHashes") List targetFQNHashes, + @Bind("tagFQNHashPrefix") String tagFQNHashPrefix); + + /** + * Batch fetch derived tags for multiple glossary term FQNs. Returns a map from glossary term + * FQN to its derived tags (tags that target that glossary term). + */ + default Map> getDerivedTagsBatch(List glossaryTermFqns) { + if (glossaryTermFqns == null || glossaryTermFqns.isEmpty()) { + return Collections.emptyMap(); + } + List tagUsages = getTagsInternalBatch(glossaryTermFqns); + Map> result = new HashMap<>(); + + for (TagLabelWithFQNHash usage : tagUsages) { + String targetFqn = usage.getTargetFQNHash(); + TagLabel tagLabel = + new TagLabel() + .withSource(TagLabel.TagSource.values()[usage.getSource()]) + .withTagFQN(usage.getTagFQN()) + .withLabelType(TagLabel.LabelType.DERIVED) + .withState(TagLabel.State.values()[usage.getState()]) + .withReason(usage.getReason()) + .withAppliedAt(usage.toTagLabel().getAppliedAt()); + if (usage.getReason() != null) { + tagLabel.withReason(usage.getReason()); + } + TagLabelUtil.applyTagCommonFieldsGracefully(tagLabel); + result.computeIfAbsent(targetFqn, k -> new ArrayList<>()).add(tagLabel); + } + return result; + } + + @ConnectionAwareSqlQuery( + value = + "SELECT tu.source, tu.tagFQN, tu.labelType, tu.targetFQNHash, tu.state, tu.reason, tu.appliedAt, tu.appliedBy, tu.metadata, " + + "CASE " + + " WHEN tu.source = 1 THEN gterm.json " + + " WHEN tu.source = 0 THEN ta.json " + + "END as json " + + "FROM tag_usage tu " + + "LEFT JOIN glossary_term_entity gterm ON tu.source = 1 AND gterm.fqnHash = tu.tagFQNHash " + + "LEFT JOIN tag ta ON tu.source = 0 AND ta.fqnHash = tu.tagFQNHash " + + "WHERE tu.targetfqnhash_lower LIKE LOWER(:targetFQNHash)", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT tu.source, tu.tagFQN, tu.labelType, tu.targetFQNHash, tu.state, tu.reason, tu.appliedAt, tu.appliedBy, tu.metadata, " + + "CASE " + + " WHEN tu.source = 1 THEN gterm.json " + + " WHEN tu.source = 0 THEN ta.json " + + "END as json " + + "FROM tag_usage tu " + + "LEFT JOIN glossary_term_entity gterm ON tu.source = 1 AND gterm.fqnHash = tu.tagFQNHash " + + "LEFT JOIN tag ta ON tu.source = 0 AND ta.fqnHash = tu.tagFQNHash " + + "WHERE tu.targetfqnhash_lower LIKE LOWER(:targetFQNHash)", + connectionType = POSTGRES) + @RegisterRowMapper(TagLabelRowMapperWithTargetFqnHash.class) + List> getTagsInternalByPrefix( + @BindConcat( + value = "targetFQNHash", + parts = {":targetFQNHashPrefix", ":postfix"}) + String... targetFQNHash); + + @SqlQuery( + "SELECT tu.source, tu.tagFQN, tu.labelType, tu.targetFQNHash, tu.state, tu.reason, tu.appliedAt, tu.appliedBy, tu.metadata, " + + "CASE " + + " WHEN tu.source = 1 THEN gterm.json " + + " WHEN tu.source = 0 THEN ta.json " + + "END as json " + + "FROM tag_usage tu " + + "LEFT JOIN glossary_term_entity gterm ON tu.source = 1 AND gterm.fqnHash = tu.tagFQNHash " + + "LEFT JOIN tag ta ON tu.source = 0 AND ta.fqnHash = tu.tagFQNHash " + + "WHERE tu.targetFQNHash IN ()") + @RegisterRowMapper(TagLabelRowMapperWithTargetFqnHash.class) + List> getTagsInternalByTargetHashes( + @BindList("targetFQNHashes") List targetFQNHashes); + + int TAG_BATCH_CHUNK_SIZE = 1000; + + default Map> getTagsByTargetFQNHashes(List targetFQNHashes) { + Map> resultSet = new LinkedHashMap<>(); + if (targetFQNHashes == null || targetFQNHashes.isEmpty()) { + return resultSet; + } + for (int i = 0; i < targetFQNHashes.size(); i += TAG_BATCH_CHUNK_SIZE) { + List chunk = + targetFQNHashes.subList(i, Math.min(i + TAG_BATCH_CHUNK_SIZE, targetFQNHashes.size())); + for (Pair pair : getTagsInternalByTargetHashes(chunk)) { + resultSet.computeIfAbsent(pair.getLeft(), k -> new ArrayList<>()).add(pair.getRight()); + } + } + return resultSet; + } + + @SqlQuery("SELECT * FROM tag_usage") + @Deprecated(since = "Release 1.1") + @RegisterRowMapper(TagLabelMapperMigration.class) + List listAll(); + + @SqlQuery( + "SELECT COUNT(*) FROM tag_usage " + + "WHERE (tagFQNHash LIKE :concatTagFQNHash OR tagFQNHash = :tagFqnHash) " + + "AND source = :source") + int getTagCount( + @Bind("source") int source, + @BindConcat( + value = "concatTagFQNHash", + original = "tagFqnHash", + parts = {":tagFqnHash", ".%"}, + hash = true) + String tagFqnHash); + + /** + * Get tag usage counts for multiple tags. + * This method retrieves counts for exact tag matches and their children in one query. + */ + @SqlQuery( + "SELECT tagFQN, count FROM (" + + " SELECT ? as tagFQN, COUNT(DISTINCT targetFQNHash) as count " + + " FROM tag_usage " + + " WHERE source = ? AND (tagFQNHash = MD5(?) OR tagFQNHash LIKE CONCAT(MD5(?), '.%'))" + + ") t WHERE tagFQN IN ()") + @RegisterRowMapper(TagCountMapper.class) + @Deprecated + List> getTagCountsBulkComplex( + @Bind("tagFQN") String sampleTagFQN, + @Bind("source") int source, + @Bind("tagFQNHash") String tagFQNHash, + @Bind("tagFQNHashPrefix") String tagFQNHashPrefix, + @BindList("tagFQNs") List tagFQNs); + + default Map getTagCountsBulk(int source, List tagFQNs) { + if (tagFQNs == null || tagFQNs.isEmpty()) { + return Collections.emptyMap(); + } + + Map resultMap = new HashMap<>(); + + // Process tags in batches to create a single efficient query + // We'll use a UNION ALL approach which is more compatible with JDBI + StringBuilder queryBuilder = new StringBuilder(); + List params = new ArrayList<>(); + + for (int i = 0; i < tagFQNs.size(); i++) { + if (i > 0) { + queryBuilder.append(" UNION ALL "); + } + queryBuilder.append( + "SELECT ? as tagFQN, COUNT(DISTINCT targetFQNHash) as count " + + "FROM tag_usage " + + "WHERE source = ? AND (tagFQNHash = MD5(?) OR tagFQNHash LIKE CONCAT(MD5(?), '.%'))"); + params.add(tagFQNs.get(i)); // tagFQN for selection + params.add(String.valueOf(source)); // source + params.add(tagFQNs.get(i)); // tagFQN for MD5 + params.add(tagFQNs.get(i)); // tagFQN for LIKE + } + + // For now, fall back to individual queries until we have a better solution + // This ensures correctness while we work on optimization + for (String tagFQN : tagFQNs) { + int count = getTagCount(source, tagFQN); + resultMap.put(tagFQN, count); + } + + return resultMap; + } + + @SqlUpdate("DELETE FROM tag_usage where targetFQNHash = :targetFQNHash") + void deleteTagsByTarget(@BindFQN("targetFQNHash") String targetFQNHash); + + @SqlUpdate( + "DELETE FROM tag_usage WHERE source = :source AND tagFQN LIKE :tagFQNPrefix AND targetFQNHash = :targetFQNHash") + void deleteTagsByPrefixAndTarget( + @Bind("source") int source, + @Bind("tagFQNPrefix") String tagFQNPrefix, + @BindFQN("targetFQNHash") String targetFQNHash); + + @SqlUpdate("DELETE FROM tag_usage WHERE targetFQNHash IN ()") + void deleteTagsByTargets(@BindListFQN("targetFQNHashes") List targetFQNs); + + @SqlUpdate( + "DELETE FROM tag_usage WHERE source = :source AND tagFQN LIKE :tagFQNPrefix AND targetFQNHash IN ()") + void deleteTagsByPrefixAndTargets( + @Bind("source") int source, + @Bind("tagFQNPrefix") String tagFQNPrefix, + @BindListFQN("targetFQNHashes") List targetFQNHashes); + + @SqlUpdate( + "DELETE FROM tag_usage where tagFQNHash = :tagFqnHash AND targetFQNHash LIKE :targetFQNHash") + void deleteTagsByTagAndTargetEntity( + @BindFQN("tagFqnHash") String tagFqnHash, + @BindConcat( + value = "targetFQNHash", + parts = {":targetFQNHashPrefix", "%"}, + hash = true) + String targetFQNHashPrefix); + + @SqlUpdate("DELETE FROM tag_usage where tagFQNHash = :tagFQNHash AND source = :source") + void deleteTagLabels(@Bind("source") int source, @BindFQN("tagFQNHash") String tagFQNHash); + + @ConnectionAwareSqlUpdate( + value = "DELETE FROM tag_usage where tagFQNHash = :tagFQNHash ORDER BY tagFQN", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "DELETE FROM tag_usage where tagFQNHash = :tagFQNHash", + connectionType = POSTGRES) + void deleteTagLabelsByFqn(@BindFQN("tagFQNHash") String tagFQNHash); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM tag_usage where targetFQNHash = :targetFQNHash OR targetFQNHash LIKE :concatTargetFQNHash ORDER BY tagFQN", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM tag_usage where targetFQNHash = :targetFQNHash OR targetFQNHash LIKE :concatTargetFQNHash", + connectionType = POSTGRES) + void deleteTagLabelsByTargetPrefix( + @BindConcat( + value = "concatTargetFQNHash", + original = "targetFQNHash", + parts = {":targetFQNHashPrefix", ".%"}, + hash = true) + String targetFQNHashPrefix); + + @Deprecated(since = "Release 1.1") + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, targetFQN)" + + "VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :targetFQN) " + + "ON DUPLICATE KEY UPDATE tagFQNHash = :tagFQNHash, targetFQNHash = :targetFQNHash", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, targetFQN) " + + "VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :targetFQN) " + + "ON CONFLICT (source, tagFQN, targetFQN) " + + "DO UPDATE SET tagFQNHash = EXCLUDED.tagFQNHash, targetFQNHash = EXCLUDED.targetFQNHash", + connectionType = POSTGRES) + void upsertFQNHash( + @Bind("source") int source, + @Bind("tagFQN") String tagFQN, + @Bind("tagFQNHash") String tagFQNHash, + @Bind("targetFQNHash") String targetFQNHash, + @Bind("labelType") int labelType, + @Bind("state") int state, + @Bind("targetFQN") String targetFQN); + + /** Update all the tagFQN starting with oldPrefix to start with newPrefix due to tag or glossary name change */ + default void updateTagPrefix(int source, String oldPrefix, String newPrefix) { + String update = + String.format( + "UPDATE tag_usage SET tagFQN = REPLACE(tagFQN, '%s.', '%s.'), tagFQNHash = REPLACE(tagFQNHash, '%s.', '%s.') WHERE source = %s AND tagFQNHash LIKE '%s.%%'", + escapeApostrophe(oldPrefix), + escapeApostrophe(newPrefix), + FullyQualifiedName.buildHash(oldPrefix), + FullyQualifiedName.buildHash(newPrefix), + source, + FullyQualifiedName.buildHash(oldPrefix)); + updateTagPrefixInternal(update); + } + + default void updateTargetFQNHashPrefix( + int source, String oldTargetFQNHashPrefix, String newTargetFQNHashPrefix) { + String update = + String.format( + "UPDATE tag_usage SET targetFQNHash = REPLACE(targetFQNHash, '%s.', '%s.') WHERE source = %s AND targetFQNHash LIKE '%s.%%'", + FullyQualifiedName.buildHash(oldTargetFQNHashPrefix), + FullyQualifiedName.buildHash(newTargetFQNHashPrefix), + source, + FullyQualifiedName.buildHash(oldTargetFQNHashPrefix)); + updateTagPrefixInternal(update); + } + + default void rename(int source, String oldFQN, String newFQN) { + renameInternal(source, oldFQN, newFQN, newFQN); // First rename tagFQN from oldFQN to newFQN + updateTagPrefix( + source, oldFQN, + newFQN); // Rename all the tagFQN prefixes starting with the oldFQN to newFQN + } + + default void renameByTargetFQNHash( + int source, String oldTargetFQNHash, String newTargetFQNHash) { + updateTargetFQNHashPrefix( + source, + oldTargetFQNHash, + newTargetFQNHash); // Rename all the targetFQN prefixes starting with the oldFQN to newFQN + } + + /** Rename the tagFQN */ + @SqlUpdate( + "Update tag_usage set tagFQN = :newFQN, tagFQNHash = :newFQNHash WHERE source = :source AND tagFQNHash = :oldFQNHash") + void renameInternal( + @Bind("source") int source, + @BindFQN("oldFQNHash") String oldFQNHash, + @Bind("newFQN") String newFQN, + @BindFQN("newFQNHash") String newFQNHash); + + @SqlUpdate( + "UPDATE tag_usage SET targetFQNHash = :newTargetFQNHash WHERE targetFQNHash = :oldTargetFQNHash") + void updateTargetFQNHash( + @BindFQN("oldTargetFQNHash") String oldTargetFQNHash, + @BindFQN("newTargetFQNHash") String newTargetFQNHash); + + @SqlUpdate("") + void updateTagPrefixInternal(@Define("update") String update); + + @SqlQuery("select targetFQNHash FROM tag_usage where tagFQNHash = :tagFQNHash") + @RegisterRowMapper(TagLabelMapper.class) + List getTargetFQNHashForTag(@BindFQN("tagFQNHash") String tagFQNHash); + + @SqlQuery("select targetFQNHash FROM tag_usage where tagFQNHash LIKE :tagFQNHash") + @RegisterRowMapper(TagLabelMapper.class) + List getTargetFQNHashForTagPrefix( + @BindConcat( + value = "tagFQNHash", + parts = {":tagFQNHashPrefix", ".%"}, + hash = true) + String tagFQNHashPrefix); + + class TagLabelMapper implements RowMapper { + @Override + public TagLabel map(ResultSet r, StatementContext ctx) throws SQLException { + TagLabelMetadata metadata = null; + try { + metadata = JsonUtils.readValue(r.getString("metadata"), TagLabelMetadata.class); + } catch (Exception e) { + // Ignore unknown fields from future schema versions — metadata is best-effort + } + return new TagLabel() + .withSource(TagLabel.TagSource.values()[r.getInt("source")]) + .withLabelType(TagLabel.LabelType.values()[r.getInt("labelType")]) + .withState(TagLabel.State.values()[r.getInt("state")]) + .withTagFQN(r.getString("tagFQN")) + .withReason(r.getString("reason")) + .withAppliedAt(r.getTimestamp("appliedAt")) + .withAppliedBy(r.getString("appliedBy")) + .withMetadata(metadata); + } + } + + class TagCountMapper implements RowMapper> { + @Override + public Map.Entry map(ResultSet r, StatementContext ctx) throws SQLException { + String tagFQN = r.getString("tagFQN"); + int count = r.getInt("count"); + return new AbstractMap.SimpleEntry<>(tagFQN, count); + } + } + + class TagLabelRowMapperWithTargetFqnHash implements RowMapper> { + @Override + public Pair map(ResultSet r, StatementContext ctx) throws SQLException { + TagLabel label = + new TagLabel() + .withSource(TagLabel.TagSource.values()[r.getInt("source")]) + .withLabelType(TagLabel.LabelType.values()[r.getInt("labelType")]) + .withState(TagLabel.State.values()[r.getInt("state")]) + .withTagFQN(r.getString("tagFQN")) + .withReason(r.getString("reason")) + .withAppliedAt(r.getTimestamp("appliedAt")) + .withAppliedBy(r.getString("appliedBy")) + .withMetadata(JsonUtils.readValue(r.getString("metadata"), TagLabelMetadata.class)); + TagLabel.TagSource source = TagLabel.TagSource.values()[r.getInt("source")]; + if (source == TagLabel.TagSource.CLASSIFICATION) { + Tag tag = JsonUtils.readValue(r.getString("json"), Tag.class); + label.setName(tag.getName()); + label.setDisplayName(tag.getDisplayName()); + label.setDescription(tag.getDescription()); + label.setStyle(tag.getStyle()); + } else if (source == TagLabel.TagSource.GLOSSARY) { + GlossaryTerm glossaryTerm = JsonUtils.readValue(r.getString("json"), GlossaryTerm.class); + label.setName(glossaryTerm.getName()); + label.setDisplayName(glossaryTerm.getDisplayName()); + label.setDescription(glossaryTerm.getDescription()); + label.setStyle(glossaryTerm.getStyle()); + } else { + throw new IllegalArgumentException("Invalid source type " + source); + } + return Pair.of(r.getString("targetFQNHash"), label); + } + } + + class TagLabelWithFQNHashMapper implements RowMapper { + @Override + public TagLabelWithFQNHash map(ResultSet rs, StatementContext ctx) throws SQLException { + TagLabelMetadata metadata = null; + try { + metadata = JsonUtils.readValue(rs.getString("metadata"), TagLabelMetadata.class); + } catch (Exception e) { + // Ignore unknown fields from future schema versions — metadata is best-effort + } + TagLabelWithFQNHash tag = new TagLabelWithFQNHash(); + tag.setTargetFQNHash(rs.getString("targetFQNHash")); + tag.setSource(rs.getInt("source")); + tag.setTagFQN(rs.getString("tagFQN")); + tag.setLabelType(rs.getInt("labelType")); + tag.setState(rs.getInt("state")); + tag.setReason(rs.getString("reason")); + tag.setAppliedAt(rs.getTimestamp("appliedAt")); + tag.setAppliedBy(rs.getString("appliedBy")); + tag.setMetadata(metadata); + return tag; + } + } + + @Getter + @Setter + class TagLabelWithFQNHash { + private String targetFQNHash; + private int source; + private String tagFQN; + private int labelType; + private int state; + private String reason; + private Date appliedAt; + private String appliedBy; + private TagLabelMetadata metadata; + + public TagLabel toTagLabel() { + TagLabel tagLabel = new TagLabel(); + TagLabel.TagSource[] sources = TagLabel.TagSource.values(); + tagLabel.setSource( + this.source >= 0 && this.source < sources.length + ? sources[this.source] + : TagLabel.TagSource.CLASSIFICATION); + tagLabel.setTagFQN(this.tagFQN); + TagLabel.LabelType[] labelTypes = TagLabel.LabelType.values(); + tagLabel.setLabelType( + this.labelType >= 0 && this.labelType < labelTypes.length + ? labelTypes[this.labelType] + : TagLabel.LabelType.MANUAL); + TagLabel.State[] states = TagLabel.State.values(); + tagLabel.setState( + this.state >= 0 && this.state < states.length + ? states[this.state] + : TagLabel.State.CONFIRMED); + tagLabel.setReason(this.reason); + tagLabel.setAppliedAt(this.appliedAt); + tagLabel.setAppliedBy(this.appliedBy); + tagLabel.setMetadata(this.metadata); + return tagLabel; + } + } + + @Getter + @Setter + @Deprecated(since = "Release 1.1") + class TagLabelMigration { + private int source; + private String tagFQN; + private String targetFQN; + private int labelType; + private int state; + private String tagFQNHash; + private String targetFQNHash; + } + + @Deprecated(since = "Release 1.1") + class TagLabelMapperMigration implements RowMapper { + @Override + public TagLabelMigration map(ResultSet r, StatementContext ctx) throws SQLException { + TagLabelMigration tagLabel = new TagLabelMigration(); + + tagLabel.setSource(r.getInt("source")); + tagLabel.setLabelType(r.getInt("labelType")); + tagLabel.setState(r.getInt("state")); + tagLabel.setTagFQN(r.getString("tagFQN")); + // TODO : Ugly , but this is present is lower version and removed on higher version + try { + // This field is removed in latest + tagLabel.setTargetFQN(r.getString("targetFQN")); + } catch (Exception ex) { + // Nothing to do + } + try { + tagLabel.setTagFQNHash(r.getString("tagFQNHash")); + } catch (Exception ex) { + // Nothing to do + } + try { + tagLabel.setTargetFQNHash(r.getString("targetFQNHash")); + } catch (Exception ex) { + // Nothing to do + } + return tagLabel; + } + } + + record TagUsageBatchRow( + int source, + String tagFQN, + String tagFQNHash, + String targetFQNHash, + int labelType, + int state, + String reason, + String appliedBy, + String metadata) {} + + record TagUsageDeleteRow(int source, String tagFQNHash, String targetFQNHash) {} + + private static String buildTagUsageKey(int source, String tagFQNHash, String targetFQNHash) { + return source + "|" + tagFQNHash + "|" + targetFQNHash; + } + + private static String buildTagUsageKey(TagUsageBatchRow row) { + return buildTagUsageKey(row.source(), row.tagFQNHash(), row.targetFQNHash()); + } + + private static String buildTagUsageKey(TagUsageDeleteRow row) { + return buildTagUsageKey(row.source(), row.tagFQNHash(), row.targetFQNHash()); + } + + Logger TAG_USAGE_LOG = LoggerFactory.getLogger(TagUsageDAO.class); + int TAG_USAGE_MAX_ATTEMPTS = 2; + AtomicLong TAG_USAGE_DEADLOCK_RETRY_COUNT = new AtomicLong(0); + + private static boolean isTransientDeadlock(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof SQLException sqlException) { + int errorCode = sqlException.getErrorCode(); + String sqlState = sqlException.getSQLState(); + if (errorCode == 1213 + || errorCode == 1205 + || "40001".equals(sqlState) + || "40P01".equals(sqlState)) { + return true; + } + } + } + return false; + } + + default void executeWithDeadlockRetry(Runnable operation) { + for (int attempt = 1; attempt <= TAG_USAGE_MAX_ATTEMPTS; attempt++) { + try { + operation.run(); + return; + } catch (RuntimeException ex) { + if (!isTransientDeadlock(ex) || attempt == TAG_USAGE_MAX_ATTEMPTS) { + throw ex; + } + try { + Thread.sleep(20L + (long) (Math.random() * 80)); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw ex; + } + long retryCount = TAG_USAGE_DEADLOCK_RETRY_COUNT.incrementAndGet(); + TAG_USAGE_LOG.debug( + "Retrying tag_usage batch after transient deadlock (attempt {}/{}), total retries={}", + attempt + 1, + TAG_USAGE_MAX_ATTEMPTS, + retryCount); + } + } + } + + /** + * Apply multiple tags in batch to improve performance + */ + default void applyTagsBatch(List tagLabels, String targetFQN) { + if (tagLabels == null || tagLabels.isEmpty()) { + return; + } + + String targetFQNHash = FullyQualifiedName.buildHash(targetFQN); + List rows = new ArrayList<>(tagLabels.size()); + + for (TagLabel tagLabel : tagLabels) { + rows.add( + new TagUsageBatchRow( + tagLabel.getSource().ordinal(), + tagLabel.getTagFQN(), + FullyQualifiedName.buildHash(tagLabel.getTagFQN()), + targetFQNHash, + tagLabel.getLabelType().ordinal(), + tagLabel.getState().ordinal(), + tagLabel.getReason(), + tagLabel.getAppliedBy(), + JsonUtils.pojoToJson(tagLabel.getMetadata()))); + } + + // De-duplicate duplicate tag applications within the same batch. + LinkedHashMap uniqueRows = new LinkedHashMap<>(rows.size()); + for (TagUsageBatchRow row : rows) { + uniqueRows.put(buildTagUsageKey(row), row); + } + rows = new ArrayList<>(uniqueRows.values()); + + // Deterministic lock acquisition order reduces deadlocks under concurrent upserts. + rows.sort( + java.util.Comparator.comparing(TagUsageBatchRow::targetFQNHash) + .thenComparing(TagUsageBatchRow::tagFQNHash) + .thenComparingInt(TagUsageBatchRow::source)); + + List sources = new ArrayList<>(rows.size()); + List tagFQNs = new ArrayList<>(rows.size()); + List tagFQNHashes = new ArrayList<>(rows.size()); + List targetFQNHashes = new ArrayList<>(rows.size()); + List labelTypes = new ArrayList<>(rows.size()); + List states = new ArrayList<>(rows.size()); + List reasons = new ArrayList<>(rows.size()); + List appliedBys = new ArrayList<>(rows.size()); + List metadataList = new ArrayList<>(rows.size()); + for (TagUsageBatchRow row : rows) { + sources.add(row.source()); + tagFQNs.add(row.tagFQN()); + tagFQNHashes.add(row.tagFQNHash()); + targetFQNHashes.add(row.targetFQNHash()); + labelTypes.add(row.labelType()); + states.add(row.state()); + reasons.add(row.reason()); + appliedBys.add(row.appliedBy()); + metadataList.add(row.metadata()); + } + + executeWithDeadlockRetry( + () -> + applyTagsBatchInternal( + sources, + tagFQNs, + tagFQNHashes, + targetFQNHashes, + labelTypes, + states, + reasons, + appliedBys, + metadataList)); + } + + /** + * Apply multiple tags in batch to multiple targets. Each entry in the map represents + * a target FQN and its associated tags. + */ + default void applyTagsBatchMultiTarget(Map> tagsByTarget) { + if (tagsByTarget == null || tagsByTarget.isEmpty()) { + return; + } + + List rows = new ArrayList<>(); + + for (Map.Entry> entry : tagsByTarget.entrySet()) { + String targetFQN = entry.getKey(); + List tagLabels = entry.getValue(); + if (tagLabels == null || tagLabels.isEmpty()) { + continue; + } + + String targetFQNHash = FullyQualifiedName.buildHash(targetFQN); + for (TagLabel tagLabel : tagLabels) { + if (tagLabel.getLabelType().equals(TagLabel.LabelType.DERIVED)) { + continue; + } + rows.add( + new TagUsageBatchRow( + tagLabel.getSource().ordinal(), + tagLabel.getTagFQN(), + FullyQualifiedName.buildHash(tagLabel.getTagFQN()), + targetFQNHash, + tagLabel.getLabelType().ordinal(), + tagLabel.getState().ordinal(), + tagLabel.getReason(), + tagLabel.getAppliedBy(), + JsonUtils.pojoToJson(tagLabel.getMetadata()))); + } + } + + if (!rows.isEmpty()) { + // De-duplicate duplicate tag applications within the same batch. + LinkedHashMap uniqueRows = new LinkedHashMap<>(rows.size()); + for (TagUsageBatchRow row : rows) { + uniqueRows.put(buildTagUsageKey(row), row); + } + rows = new ArrayList<>(uniqueRows.values()); + + // Deterministic lock acquisition order reduces deadlocks under concurrent upserts. + rows.sort( + java.util.Comparator.comparing(TagUsageBatchRow::targetFQNHash) + .thenComparing(TagUsageBatchRow::tagFQNHash) + .thenComparingInt(TagUsageBatchRow::source)); + + List sources = new ArrayList<>(rows.size()); + List tagFQNs = new ArrayList<>(rows.size()); + List tagFQNHashes = new ArrayList<>(rows.size()); + List targetFQNHashes = new ArrayList<>(rows.size()); + List labelTypes = new ArrayList<>(rows.size()); + List states = new ArrayList<>(rows.size()); + List reasons = new ArrayList<>(rows.size()); + List appliedBys = new ArrayList<>(rows.size()); + List metadataList = new ArrayList<>(rows.size()); + for (TagUsageBatchRow row : rows) { + sources.add(row.source()); + tagFQNs.add(row.tagFQN()); + tagFQNHashes.add(row.tagFQNHash()); + targetFQNHashes.add(row.targetFQNHash()); + labelTypes.add(row.labelType()); + states.add(row.state()); + reasons.add(row.reason()); + appliedBys.add(row.appliedBy()); + metadataList.add(row.metadata()); + } + + executeWithDeadlockRetry( + () -> + applyTagsBatchInternal( + sources, + tagFQNs, + tagFQNHashes, + targetFQNHashes, + labelTypes, + states, + reasons, + appliedBys, + metadataList)); + } + } + + @Transaction + @ConnectionAwareSqlBatch( + value = + "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedBy, metadata) " + + "VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :reason, :appliedBy, :metadata) " + + "ON DUPLICATE KEY UPDATE labelType = VALUES(labelType), state = VALUES(state), reason = VALUES(reason), metadata = VALUES(metadata)", + connectionType = MYSQL) + @ConnectionAwareSqlBatch( + value = + "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedBy, metadata) " + + "VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :reason, :appliedBy, :metadata :: jsonb) " + + "ON CONFLICT (source, tagFQNHash, targetFQNHash) DO UPDATE SET labelType = EXCLUDED.labelType, " + + "state = EXCLUDED.state, reason = EXCLUDED.reason, metadata = EXCLUDED.metadata", + connectionType = POSTGRES) + void applyTagsBatchInternal( + @Bind("source") List sources, + @Bind("tagFQN") List tagFQNs, + @Bind("tagFQNHash") List tagFQNHashes, + @Bind("targetFQNHash") List targetFQNHashes, + @Bind("labelType") List labelTypes, + @Bind("state") List states, + @Bind("reason") List reasons, + @Bind("appliedBy") List appliedBys, + @Bind("metadata") List metadataList); + + /** + * Delete multiple tags in batch to improve performance + */ + default void deleteTagsBatch(List tagLabels, String targetFQN) { + if (tagLabels == null || tagLabels.isEmpty()) { + return; + } + + String targetFQNHash = FullyQualifiedName.buildHash(targetFQN); + List rows = new ArrayList<>(tagLabels.size()); + + for (TagLabel tagLabel : tagLabels) { + rows.add( + new TagUsageDeleteRow( + tagLabel.getSource().ordinal(), + FullyQualifiedName.buildHash(tagLabel.getTagFQN()), + targetFQNHash)); + } + + LinkedHashMap uniqueRows = new LinkedHashMap<>(rows.size()); + for (TagUsageDeleteRow row : rows) { + uniqueRows.put(buildTagUsageKey(row), row); + } + rows = new ArrayList<>(uniqueRows.values()); + + rows.sort( + java.util.Comparator.comparing(TagUsageDeleteRow::targetFQNHash) + .thenComparing(TagUsageDeleteRow::tagFQNHash) + .thenComparingInt(TagUsageDeleteRow::source)); + + List sources = new ArrayList<>(rows.size()); + List tagFQNHashes = new ArrayList<>(rows.size()); + List targetFQNHashes = new ArrayList<>(rows.size()); + for (TagUsageDeleteRow row : rows) { + sources.add(row.source()); + tagFQNHashes.add(row.tagFQNHash()); + targetFQNHashes.add(row.targetFQNHash()); + } + + executeWithDeadlockRetry( + () -> deleteTagsBatchInternal(sources, tagFQNHashes, targetFQNHashes)); + } + + @Transaction + @ConnectionAwareSqlBatch( + value = + "DELETE FROM tag_usage WHERE source = :source AND tagFQNHash = :tagFQNHash AND targetFQNHash = :targetFQNHash", + connectionType = MYSQL) + @ConnectionAwareSqlBatch( + value = + "DELETE FROM tag_usage WHERE source = :source AND tagFQNHash = :tagFQNHash AND targetFQNHash = :targetFQNHash", + connectionType = POSTGRES) + void deleteTagsBatchInternal( + @Bind("source") List sources, + @Bind("tagFQNHash") List tagFQNHashes, + @Bind("targetFQNHash") List targetFQNHashes); + + @SqlQuery("SELECT COUNT(*) FROM tag_usage") + long getTotalTagUsageCount(); + + @SqlQuery( + "SELECT source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedAt, appliedBy, metadata FROM tag_usage ORDER BY source, tagFQNHash LIMIT :limit OFFSET :offset") + @RegisterRowMapper(TagUsageObjectMapper.class) + List getAllTagUsagesPaginated( + @Bind("offset") long offset, @Bind("limit") int limit); + + @SqlUpdate( + "DELETE FROM tag_usage WHERE source = :source AND tagFQNHash = :tagFQNHash AND targetFQNHash = :targetFQNHash") + int deleteTagUsage( + @Bind("source") int source, + @Bind("tagFQNHash") String tagFQNHash, + @Bind("targetFQNHash") String targetFQNHash); + } + + @Getter + @Builder + class TagUsageObject { + private int source; + private String tagFQN; + private String tagFQNHash; + private String targetFQNHash; + private int labelType; + private int state; + private String reason; + private Date appliedAt; + private String appliedBy; + private TagLabelMetadata metadata; + } + + class TagUsageObjectMapper implements RowMapper { + @Override + public TagUsageObject map(ResultSet r, StatementContext ctx) throws SQLException { + return TagUsageObject.builder() + .source(r.getInt("source")) + .tagFQN(r.getString("tagFQN")) + .tagFQNHash(r.getString("tagFQNHash")) + .targetFQNHash(r.getString("targetFQNHash")) + .labelType(r.getInt("labelType")) + .state(r.getInt("state")) + .reason(r.getString("reason")) + .appliedAt(r.getTimestamp("appliedAt")) + .appliedBy(r.getString("appliedBy")) + .metadata(JsonUtils.readValue(r.getString("metadata"), TagLabelMetadata.class)) + .build(); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 89cbfc86ad16..f91d5061d31e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -13,15150 +13,368 @@ package org.openmetadata.service.jdbi3; -import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.schema.type.Relationship.CONTAINS; -import static org.openmetadata.schema.type.Relationship.HAS; -import static org.openmetadata.schema.type.Relationship.MENTIONED_IN; -import static org.openmetadata.schema.type.Relationship.OWNS; -import static org.openmetadata.service.Entity.APPLICATION; -import static org.openmetadata.service.Entity.GLOSSARY_TERM; -import static org.openmetadata.service.Entity.ORGANIZATION_NAME; -import static org.openmetadata.service.Entity.QUERY; -import static org.openmetadata.service.Entity.TEAM; -import static org.openmetadata.service.Entity.USER; -import static org.openmetadata.service.jdbi3.ListFilter.escapeApostrophe; import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Collectors; -import lombok.Builder; -import lombok.Getter; -import lombok.Setter; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.commons.lang3.tuple.Triple; import org.jdbi.v3.core.mapper.RowMapper; import org.jdbi.v3.core.statement.StatementContext; -import org.jdbi.v3.core.statement.StatementException; import org.jdbi.v3.sqlobject.CreateSqlObject; -import org.jdbi.v3.sqlobject.config.RegisterRowMapper; import org.jdbi.v3.sqlobject.customizer.Bind; -import org.jdbi.v3.sqlobject.customizer.BindBean; -import org.jdbi.v3.sqlobject.customizer.BindBeanList; -import org.jdbi.v3.sqlobject.customizer.BindList; -import org.jdbi.v3.sqlobject.customizer.BindMap; -import org.jdbi.v3.sqlobject.customizer.BindMethods; -import org.jdbi.v3.sqlobject.customizer.Define; -import org.jdbi.v3.sqlobject.statement.SqlBatch; import org.jdbi.v3.sqlobject.statement.SqlQuery; import org.jdbi.v3.sqlobject.statement.SqlUpdate; -import org.jdbi.v3.sqlobject.statement.UseRowMapper; -import org.jdbi.v3.sqlobject.transaction.Transaction; -import org.openmetadata.api.configuration.UiThemePreference; -import org.openmetadata.schema.TokenInterface; -import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.schema.analytics.WebAnalyticEvent; -import org.openmetadata.schema.api.configuration.LoginConfiguration; -import org.openmetadata.schema.api.configuration.MCPConfiguration; -import org.openmetadata.schema.api.configuration.OpenMetadataBaseUrlConfiguration; -import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration; -import org.openmetadata.schema.api.lineage.LineageSettings; -import org.openmetadata.schema.api.search.SearchSettings; -import org.openmetadata.schema.api.security.AuthenticationConfiguration; -import org.openmetadata.schema.api.security.AuthorizerConfiguration; -import org.openmetadata.schema.auth.EmailVerificationToken; -import org.openmetadata.schema.auth.PasswordResetToken; -import org.openmetadata.schema.auth.PersonalAccessToken; -import org.openmetadata.schema.auth.RefreshToken; -import org.openmetadata.schema.auth.TokenType; -import org.openmetadata.schema.auth.collate.SupportToken; -import org.openmetadata.schema.configuration.AssetCertificationSettings; -import org.openmetadata.schema.configuration.EntityRulesSettings; -import org.openmetadata.schema.configuration.GlossaryTermRelationSettings; -import org.openmetadata.schema.configuration.OpenLineageSettings; -import org.openmetadata.schema.configuration.WorkflowSettings; -import org.openmetadata.schema.dataInsight.DataInsightChart; -import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; -import org.openmetadata.schema.dataInsight.kpi.Kpi; -import org.openmetadata.schema.email.SmtpSettings; -import org.openmetadata.schema.entities.docStore.Document; -import org.openmetadata.schema.entity.Bot; -import org.openmetadata.schema.entity.Type; -import org.openmetadata.schema.entity.app.App; -import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; -import org.openmetadata.schema.entity.automations.Workflow; -import org.openmetadata.schema.entity.classification.Classification; -import org.openmetadata.schema.entity.classification.Tag; -import org.openmetadata.schema.entity.context.ContextMemory; -import org.openmetadata.schema.entity.data.APICollection; -import org.openmetadata.schema.entity.data.APIEndpoint; -import org.openmetadata.schema.entity.data.Chart; -import org.openmetadata.schema.entity.data.Container; -import org.openmetadata.schema.entity.data.Dashboard; -import org.openmetadata.schema.entity.data.DashboardDataModel; -import org.openmetadata.schema.entity.data.DataContract; -import org.openmetadata.schema.entity.data.Database; -import org.openmetadata.schema.entity.data.DatabaseSchema; -import org.openmetadata.schema.entity.data.Directory; -import org.openmetadata.schema.entity.data.File; -import org.openmetadata.schema.entity.data.Glossary; -import org.openmetadata.schema.entity.data.GlossaryTerm; -import org.openmetadata.schema.entity.data.Metric; -import org.openmetadata.schema.entity.data.MlModel; -import org.openmetadata.schema.entity.data.Pipeline; -import org.openmetadata.schema.entity.data.Query; -import org.openmetadata.schema.entity.data.Report; -import org.openmetadata.schema.entity.data.SearchIndex; -import org.openmetadata.schema.entity.data.Spreadsheet; -import org.openmetadata.schema.entity.data.StoredProcedure; -import org.openmetadata.schema.entity.data.Table; -import org.openmetadata.schema.entity.data.Topic; -import org.openmetadata.schema.entity.data.Worksheet; -import org.openmetadata.schema.entity.domains.DataProduct; -import org.openmetadata.schema.entity.domains.Domain; -import org.openmetadata.schema.entity.events.EventSubscription; -import org.openmetadata.schema.entity.events.FailedEvent; -import org.openmetadata.schema.entity.events.FailedEventResponse; -import org.openmetadata.schema.entity.events.NotificationTemplate; -import org.openmetadata.schema.entity.feed.Announcement; -import org.openmetadata.schema.entity.feed.TaskFormSchema; -import org.openmetadata.schema.entity.learning.LearningResource; -import org.openmetadata.schema.entity.policies.Policy; -import org.openmetadata.schema.entity.services.ApiService; -import org.openmetadata.schema.entity.services.DashboardService; -import org.openmetadata.schema.entity.services.DatabaseService; -import org.openmetadata.schema.entity.services.DriveService; -import org.openmetadata.schema.entity.services.MessagingService; -import org.openmetadata.schema.entity.services.MetadataService; -import org.openmetadata.schema.entity.services.MlModelService; -import org.openmetadata.schema.entity.services.PipelineService; -import org.openmetadata.schema.entity.services.SearchService; -import org.openmetadata.schema.entity.services.SecurityService; -import org.openmetadata.schema.entity.services.StorageService; -import org.openmetadata.schema.entity.services.connections.TestConnectionDefinition; -import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; -import org.openmetadata.schema.entity.tasks.Task; -import org.openmetadata.schema.entity.teams.Persona; -import org.openmetadata.schema.entity.teams.Role; -import org.openmetadata.schema.entity.teams.Team; -import org.openmetadata.schema.entity.teams.User; -import org.openmetadata.schema.governance.workflows.WorkflowDefinition; -import org.openmetadata.schema.security.scim.ScimConfiguration; -import org.openmetadata.schema.service.configuration.teamsApp.TeamsAppConfiguration; -import org.openmetadata.schema.settings.Settings; -import org.openmetadata.schema.settings.SettingsType; -import org.openmetadata.schema.tests.TestCase; -import org.openmetadata.schema.tests.TestDefinition; -import org.openmetadata.schema.tests.TestSuite; -import org.openmetadata.schema.type.ChangeEvent; -import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.type.EventType; -import org.openmetadata.schema.type.Include; -import org.openmetadata.schema.type.Relationship; -import org.openmetadata.schema.type.TagLabel; -import org.openmetadata.schema.type.TagLabelMetadata; -import org.openmetadata.schema.type.UsageDetails; -import org.openmetadata.schema.type.UsageStats; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; -import org.openmetadata.schema.utils.EntityInterfaceUtil; -import org.openmetadata.schema.utils.JsonUtils; -import org.openmetadata.service.Entity; -import org.openmetadata.service.audit.AuditLogRecord; -import org.openmetadata.service.audit.AuditLogRecordMapper; -import org.openmetadata.service.jdbi3.CollectionDAO.TagUsageDAO.TagLabelMapper; -import org.openmetadata.service.jdbi3.CollectionDAO.UsageDAO.UsageDetailsMapper; -import org.openmetadata.service.jdbi3.FeedRepository.FilterType; -import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlBatch; -import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; -import org.openmetadata.service.jdbi3.oauth.OAuthRecords; -import org.openmetadata.service.resources.events.subscription.TypedEvent; -import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; -import org.openmetadata.service.resources.tags.TagLabelUtil; -import org.openmetadata.service.security.session.UserSession; -import org.openmetadata.service.util.EntityUtil; -import org.openmetadata.service.util.FullyQualifiedName; -import org.openmetadata.service.util.jdbi.BindConcat; import org.openmetadata.service.util.jdbi.BindFQN; -import org.openmetadata.service.util.jdbi.BindJsonContains; -import org.openmetadata.service.util.jdbi.BindListFQN; -import org.openmetadata.service.util.jdbi.BindUUID; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public interface CollectionDAO { +public interface CollectionDAO + extends CoreRelationshipDAOs, + OAuthDAOs, + WorkflowDocStoreDAOs, + AccessControlDAOs, + EntityDataDAOs, + DataAssetServiceDAOs, + SystemTokenDAOs, + KnowledgeAssetDAOs, + EventSubscriptionDAOs, + GovernanceDAOs, + ActivityAuditDAOs, + TimeSeriesDAOs, + ClassificationTagDAOs, + FeedDAOs, + AiGovernanceDAOs, + SearchReindexDAOs, + RdfInfraDAOs { @CreateSqlObject - DatabaseDAO databaseDAO(); - - @CreateSqlObject - DatabaseSchemaDAO databaseSchemaDAO(); - - @CreateSqlObject - EntityRelationshipDAO relationshipDAO(); - - @CreateSqlObject - FieldRelationshipDAO fieldRelationshipDAO(); - - @CreateSqlObject - EntityExtensionDAO entityExtensionDAO(); - - @CreateSqlObject - AppExtensionTimeSeries appExtensionTimeSeriesDao(); - - @CreateSqlObject - AppsDataStore appStoreDAO(); - - @CreateSqlObject - EntityExtensionTimeSeriesDAO entityExtensionTimeSeriesDao(); - - @CreateSqlObject - ReportDataTimeSeriesDAO reportDataTimeSeriesDao(); - - @CreateSqlObject - ProfilerDataTimeSeriesDAO profilerDataTimeSeriesDao(); - - @CreateSqlObject - IndexMappingVersionDAO indexMappingVersionDAO(); - - @CreateSqlObject - DataQualityDataTimeSeriesDAO dataQualityDataTimeSeriesDao(); - - @CreateSqlObject - TestCaseResolutionStatusTimeSeriesDAO testCaseResolutionStatusTimeSeriesDao(); - - @CreateSqlObject - QueryCostTimeSeriesDAO queryCostRecordTimeSeriesDAO(); - - @CreateSqlObject - TestCaseResultTimeSeriesDAO testCaseResultTimeSeriesDao(); - - @CreateSqlObject - TestCaseDimensionResultTimeSeriesDAO testCaseDimensionResultTimeSeriesDao(); - - @CreateSqlObject - RoleDAO roleDAO(); - - @CreateSqlObject - UserDAO userDAO(); - - @CreateSqlObject - TeamDAO teamDAO(); - - @CreateSqlObject - PersonaDAO personaDAO(); - - @CreateSqlObject - TagUsageDAO tagUsageDAO(); - - @CreateSqlObject - TagDAO tagDAO(); - - @CreateSqlObject - ClassificationDAO classificationDAO(); - - @CreateSqlObject - TableDAO tableDAO(); - - @CreateSqlObject - QueryDAO queryDAO(); - - @CreateSqlObject - UsageDAO usageDAO(); - - @CreateSqlObject - MetricDAO metricDAO(); - - @CreateSqlObject - ChartDAO chartDAO(); - - @CreateSqlObject - ApplicationDAO applicationDAO(); - - @CreateSqlObject - ApplicationMarketPlaceDAO applicationMarketPlaceDAO(); - - @CreateSqlObject - PipelineDAO pipelineDAO(); - - @CreateSqlObject - DashboardDAO dashboardDAO(); - - @CreateSqlObject - ReportDAO reportDAO(); - - @CreateSqlObject - TopicDAO topicDAO(); - - @CreateSqlObject - MlModelDAO mlModelDAO(); - - @CreateSqlObject - SearchIndexDAO searchIndexDAO(); - - @CreateSqlObject - GlossaryDAO glossaryDAO(); - - @CreateSqlObject - GlossaryTermDAO glossaryTermDAO(); - - @CreateSqlObject - BotDAO botDAO(); - - @CreateSqlObject - DomainDAO domainDAO(); - - @CreateSqlObject - DataProductDAO dataProductDAO(); - - @CreateSqlObject - DataContractDAO dataContractDAO(); - - @CreateSqlObject - EventSubscriptionDAO eventSubscriptionDAO(); - - @CreateSqlObject - NotificationTemplateDAO notificationTemplateDAO(); - - @CreateSqlObject - PolicyDAO policyDAO(); - - @CreateSqlObject - IngestionPipelineDAO ingestionPipelineDAO(); - - @CreateSqlObject - DatabaseServiceDAO dbServiceDAO(); - - @CreateSqlObject - MetadataServiceDAO metadataServiceDAO(); - - @CreateSqlObject - PipelineServiceDAO pipelineServiceDAO(); - - @CreateSqlObject - MlModelServiceDAO mlModelServiceDAO(); - - @CreateSqlObject - DashboardServiceDAO dashboardServiceDAO(); - - @CreateSqlObject - MessagingServiceDAO messagingServiceDAO(); - - @CreateSqlObject - StorageServiceDAO storageServiceDAO(); - - @CreateSqlObject - SearchServiceDAO searchServiceDAO(); - - @CreateSqlObject - SecurityServiceDAO securityServiceDAO(); - - @CreateSqlObject - ApiServiceDAO apiServiceDAO(); - - @CreateSqlObject - DriveServiceDAO driveServiceDAO(); - - @CreateSqlObject - ContainerDAO containerDAO(); - - @CreateSqlObject - DirectoryDAO directoryDAO(); - - @CreateSqlObject - FileDAO fileDAO(); - - @CreateSqlObject - SpreadsheetDAO spreadsheetDAO(); - - @CreateSqlObject - WorksheetDAO worksheetDAO(); - - @CreateSqlObject - FolderDAO folderDAO(); - - @CreateSqlObject - ContextFileDAO contextFileDAO(); - - @CreateSqlObject - ContextFileContentDAO contextFileContentDAO(); - - @CreateSqlObject - KnowledgePageDAO knowledgePageDAO(); - - @CreateSqlObject - AssetDAO assetDAO(); - - @CreateSqlObject - FeedDAO feedDAO(); - - @CreateSqlObject - TaskDAO taskDAO(); - - @CreateSqlObject - AnnouncementDAO announcementDAO(); - - @CreateSqlObject - TaskFormSchemaDAO taskFormSchemaDAO(); - - @CreateSqlObject - StoredProcedureDAO storedProcedureDAO(); - - @CreateSqlObject - ChangeEventDAO changeEventDAO(); - - @CreateSqlObject - TypeEntityDAO typeEntityDAO(); - - @CreateSqlObject - TestDefinitionDAO testDefinitionDAO(); - - @CreateSqlObject - TestConnectionDefinitionDAO testConnectionDefinitionDAO(); - - @CreateSqlObject - TestSuiteDAO testSuiteDAO(); - - @CreateSqlObject - TestCaseDAO testCaseDAO(); - - @CreateSqlObject - WebAnalyticEventDAO webAnalyticEventDAO(); - - @CreateSqlObject - DataInsightCustomChartDAO dataInsightCustomChartDAO(); - - @CreateSqlObject - DataInsightChartDAO dataInsightChartDAO(); - - @CreateSqlObject - SystemDAO systemDAO(); - - @CreateSqlObject - TokenDAO getTokenDAO(); - - @CreateSqlObject - UserSessionDAO getUserSessionDAO(); - - @CreateSqlObject - KpiDAO kpiDAO(); - - @CreateSqlObject - WorkflowDAO workflowDAO(); - - @CreateSqlObject - DataModelDAO dashboardDataModelDAO(); - - @CreateSqlObject - DocStoreDAO docStoreDAO(); - - @CreateSqlObject - LearningResourceDAO learningResourceDAO(); - - @CreateSqlObject - ContextMemoryDAO contextMemoryDAO(); - - @CreateSqlObject - SuggestionDAO suggestionDAO(); - - @CreateSqlObject - APICollectionDAO apiCollectionDAO(); - - @CreateSqlObject - APIEndpointDAO apiEndpointDAO(); - - @CreateSqlObject - WorkflowDefinitionDAO workflowDefinitionDAO(); - - @CreateSqlObject - WorkflowInstanceTimeSeriesDAO workflowInstanceTimeSeriesDAO(); - - @CreateSqlObject - WorkflowInstanceStateTimeSeriesDAO workflowInstanceStateTimeSeriesDAO(); - - @CreateSqlObject - DeletionLockDAO deletionLockDAO(); - - @CreateSqlObject - RecognizerFeedbackDAO recognizerFeedbackDAO(); - - @CreateSqlObject - AIApplicationDAO aiApplicationDAO(); - - @CreateSqlObject - LLMModelDAO llmModelDAO(); - - @CreateSqlObject - PromptTemplateDAO promptTemplateDAO(); - - @CreateSqlObject - AgentExecutionDAO agentExecutionDAO(); - - @CreateSqlObject - AIGovernancePolicyDAO aiGovernancePolicyDAO(); - - @CreateSqlObject - McpServerDAO mcpServerDAO(); - - @CreateSqlObject - McpExecutionDAO mcpExecutionDAO(); - - @CreateSqlObject - LLMServiceDAO llmServiceDAO(); - - @CreateSqlObject - ActivityStreamDAO activityStreamDAO(); - - @CreateSqlObject - ActivityStreamConfigDAO activityStreamConfigDAO(); - - @CreateSqlObject - McpServiceDAO mcpServiceDAO(); - - @CreateSqlObject - SearchIndexJobDAO searchIndexJobDAO(); - - @CreateSqlObject - SearchIndexPartitionDAO searchIndexPartitionDAO(); - - @CreateSqlObject - SearchReindexLockDAO searchReindexLockDAO(); - - @CreateSqlObject - SearchIndexFailureDAO searchIndexFailureDAO(); - - @CreateSqlObject - SearchIndexRetryQueueDAO searchIndexRetryQueueDAO(); - - @CreateSqlObject - SearchIndexServerStatsDAO searchIndexServerStatsDAO(); - - @CreateSqlObject - RdfIndexJobDAO rdfIndexJobDAO(); - - @CreateSqlObject - RdfIndexPartitionDAO rdfIndexPartitionDAO(); - - @CreateSqlObject - RdfReindexLockDAO rdfReindexLockDAO(); - - @CreateSqlObject - RdfIndexServerStatsDAO rdfIndexServerStatsDAO(); - - @CreateSqlObject - AuditLogDAO auditLogDAO(); - - @CreateSqlObject - OAuthClientDAO oauthClientDAO(); - - @CreateSqlObject - OAuthAuthorizationCodeDAO oauthAuthorizationCodeDAO(); - - @CreateSqlObject - OAuthAccessTokenDAO oauthAccessTokenDAO(); - - @CreateSqlObject - OAuthRefreshTokenDAO oauthRefreshTokenDAO(); - - @CreateSqlObject - McpPendingAuthRequestDAO mcpPendingAuthRequestDAO(); - - interface DashboardDAO extends EntityDAO { - @Override - default String getTableName() { - return "dashboard_entity"; - } - - @Override - default Class getEntityClass() { - return Dashboard.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface DashboardServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "dashboard_service_entity"; - } - - @Override - default Class getEntityClass() { - return DashboardService.class; - } - } - - interface DatabaseDAO extends EntityDAO { - @Override - default String getTableName() { - return "database_entity"; - } - - @Override - default Class getEntityClass() { - return Database.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @ConnectionAwareSqlQuery( - value = - "select JSON_EXTRACT(json, '$.fullyQualifiedName') from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "select json ->> 'fullyQualifiedName' from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')", - connectionType = POSTGRES) - List getBrokenDatabase(); - - @SqlUpdate( - value = - "delete from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')") - int removeDatabase(); - } - - interface DatabaseSchemaDAO extends EntityDAO { - @Override - default String getTableName() { - return "database_schema_entity"; - } - - @Override - default Class getEntityClass() { - return DatabaseSchema.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @ConnectionAwareSqlQuery( - value = - "select JSON_EXTRACT(json, '$.fullyQualifiedName') from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "select json ->> 'fullyQualifiedName' from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')", - connectionType = POSTGRES) - List getBrokenDatabaseSchemas(); - - @SqlUpdate( - value = - "delete from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')") - int removeBrokenDatabaseSchemas(); - } - - interface DatabaseServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "dbservice_entity"; - } - - @Override - default Class getEntityClass() { - return DatabaseService.class; - } - } - - interface MetadataServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "metadata_service_entity"; - } - - @Override - default Class getEntityClass() { - return MetadataService.class; - } - } - - interface TestConnectionDefinitionDAO extends EntityDAO { - @Override - default String getTableName() { - return "test_connection_definition"; - } - - @Override - default Class getEntityClass() { - return TestConnectionDefinition.class; - } - } - - interface StorageServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "storage_service_entity"; - } - - @Override - default Class getEntityClass() { - return StorageService.class; - } - } - - interface ContainerDAO extends EntityDAO { - @Override - default String getTableName() { - return "storage_container_entity"; - } - - @Override - default Class getEntityClass() { - return Container.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - - // By default, root will be false. We won't filter the results then - if (!root) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - - // Distinct method name (listRootBefore) is required: a same-signature `listBefore` - // here would override EntityDAO's default `listBefore(String, Map, String, int, - // String, String)` and make every non-root list call also pick up the depth check, - // silently filtering out child containers from generic `?service=...` listings. - return listRootBefore( - getTableName(), rootListingParams(filter), condition, limit, beforeName, beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - - if (!root) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - - return listRootAfter( - getTableName(), rootListingParams(filter), condition, limit, afterName, afterId); - } - - @Override - default int listCount(ListFilter filter) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - - if (!root) { - return EntityDAO.super.listCount(filter); - } - - return listRootCount( - getTableName(), getNameHashColumn(), rootListingParams(filter), condition); - } - - /** - * Build the bind map the listRoot SQL expects. The depth predicate - * ({@code fqnHash NOT LIKE :serviceHashChild}) needs the {@code serviceHashChild} - * bind to be set on every call, but {@link ListFilter#getServiceCondition} only - * adds it when {@code ?service=} is present. For the {@code ?root=true} case - * without a service filter — "all root containers across all services" — - * we default the bind to {@code %.%.%}, which excludes any fqnHash with two or more - * separators (everything strictly below the immediate level). Index usage is naturally - * weaker here since the prefix LIKE is also absent, but no-service root listings are - * rare and the result is at most one row per service. - */ - private static java.util.Map rootListingParams(ListFilter filter) { - java.util.Map params = new java.util.HashMap<>(filter.getQueryParams()); - params.putIfAbsent("serviceHashChild", "%.%.%"); - return params; - } - - // Root-only listing (?root=true) returns containers that are direct children of the - // service — i.e. one segment below the service in the FQN tree. - // - // Earlier implementations relied on `entity_relationship` as the source of truth ("a - // container is a root iff no inbound CONTAINS edge exists"). That broke under two - // separate failure modes: - // 1. Connectors (and bulk imports) that create deeply-nested containers without - // writing the parent CONTAINS edge — those orphans satisfy "no inbound edge" and - // surface at the service root, even though their FQN is many segments deep. The - // breadcrumb UI (which reads the FQN) and the listing (which reads the relationship) - // disagreed about where the container lived. - // 2. The NOT EXISTS anti-join needed a composite (fromEntity, toEntity, relation, toId) - // index to be cheap; under pgjdbc generic plans the planner often chose the - // ORDER BY index instead, falling back to a full-table scan and making the count - // query 1-2s on a service with hundreds of thousands of containers. - // - // The FQN is the canonical hierarchy in OpenMetadata (it's set unconditionally at write - // time and is what the breadcrumb UI consumes). `fqnHash` is built by joining - // fixed-width MD5 segments with '.', so depth follows from the count of separators — - // a direct child of the service has a fqnHash matching `.<32hex>` and - // contains no further '.'. We express "not a direct child" as `fqnHash LIKE - // .%.%` and reject those rows. ListFilter.getFqnPrefixCondition binds - // both `:serviceHash` (already used by the prefix LIKE in ) and - // `:serviceHashChild` (the `.%.%` companion) so the SQL just plugs them in. - @SqlQuery( - value = - "SELECT json FROM (" - + "SELECT name, id, ce.json FROM

ce " - + " AND " - + "ce.fqnHash NOT LIKE :serviceHashChild AND " - + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " - + "ORDER BY name DESC, id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name, id") - List listRootBefore( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId); - - @SqlQuery( - value = - "SELECT ce.json FROM
ce " - + " AND " - + "ce.fqnHash NOT LIKE :serviceHashChild AND " - + "(name > :afterName OR (name = :afterName AND id > :afterId)) " - + "ORDER BY name, id " - + "LIMIT :limit") - List listRootAfter( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId); - - @ConnectionAwareSqlQuery( - value = - "SELECT count() FROM
ce " - + " AND ce.fqnHash NOT LIKE :serviceHashChild", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM
ce " - + " AND ce.fqnHash NOT LIKE :serviceHashChild", - connectionType = POSTGRES) - int listRootCount( - @Define("table") String table, - @Define("nameHashColumn") String nameHashColumn, - @BindMap Map params, - @Define("sqlCondition") String mysqlCond); - - /** - * Lightweight projection used by paginated children listings. Pulls only the columns the - * UI's children table needs (id, name, displayName, fqn, description) plus the soft-delete - * flag. Skips JSON deserialization of heavy fields like {@code dataModel}, {@code tags}, - * and {@code owners} which can each carry MBs of column-schema metadata for parquet - * containers. The service reference is restored separately by - * {@link ContainerRepository#fetchAndSetDefaultService(java.util.List)}. - */ - @ConnectionAwareSqlQuery( - value = - "SELECT id, name, " - + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.displayName')) AS displayName, " - + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.fullyQualifiedName')) AS fqn, " - + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.description')) AS description, " - + "deleted " - + "FROM storage_container_entity WHERE id IN ()", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT id, name, " - + "json->>'displayName' AS displayName, " - + "json->>'fullyQualifiedName' AS fqn, " - + "json->>'description' AS description, " - + "deleted " - + "FROM storage_container_entity WHERE id IN ()", - connectionType = POSTGRES) - @RegisterRowMapper(ContainerSummaryRowMapper.class) - List findContainerSummaryRows(@BindList("ids") List ids); - - default List findContainerSummariesByIds(List ids) { - if (ids == null || ids.isEmpty()) { - return List.of(); - } - List idStrings = ids.stream().map(UUID::toString).distinct().toList(); - int maxChunkSize = 30000; - if (idStrings.size() <= maxChunkSize) { - return findContainerSummaryRows(idStrings); - } - List all = new ArrayList<>(idStrings.size()); - for (int i = 0; i < idStrings.size(); i += maxChunkSize) { - List chunk = idStrings.subList(i, Math.min(i + maxChunkSize, idStrings.size())); - all.addAll(findContainerSummaryRows(chunk)); - } - return all; - } - - // FQN-based direct-children page. The two binds (`:parentHash` = '.%' and - // `:parentHashChild` = '.%.%') together select containers whose FQN is exactly one - // segment below the parent — same shape used by the root listing in listRootAfter, just - // without the cursor pagination. Returns the slim projection used by the children table - // UI; the caller restores the service reference separately. `:includeDeleted` is a - // tri-state: 'NON_DELETED' (default), 'DELETED', or 'ALL'. `:nameLike` is a LIKE pattern - // applied to LOWER(name); callers pass '%' for "no filter" or '%%' - // for a substring search. ESCAPE '!' is set explicitly so the same pattern semantics - // hold on MySQL (default escape is '\') and PostgreSQL (default has no escape char). - // '!' is preferred over '\' because the JDBI ColonPrefixSqlParser scans string literals - // to skip ':' bind markers inside them, and a literal {@code '\'} confuses the scanner - // (it treats the trailing backslash as an escape and consumes the closing quote), - // leaving a downstream {@code :includeDeleted} bind un-substituted and the prepared - // statement malformed. - @ConnectionAwareSqlQuery( - value = - "SELECT id, name, " - + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.displayName')) AS displayName, " - + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.fullyQualifiedName')) AS fqn, " - + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.description')) AS description, " - + "deleted " - + "FROM storage_container_entity " - + "WHERE fqnHash LIKE :parentHash AND fqnHash NOT LIKE :parentHashChild " - + " AND LOWER(name) LIKE :nameLike ESCAPE '!' " - + " AND (:includeDeleted = 'ALL' " - + " OR (:includeDeleted = 'DELETED' AND deleted = TRUE) " - + " OR (:includeDeleted = 'NON_DELETED' AND deleted = FALSE)) " - + "ORDER BY name, id LIMIT :limit OFFSET :offset", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT id, name, " - + "json->>'displayName' AS displayName, " - + "json->>'fullyQualifiedName' AS fqn, " - + "json->>'description' AS description, " - + "deleted " - + "FROM storage_container_entity " - + "WHERE fqnHash LIKE :parentHash AND fqnHash NOT LIKE :parentHashChild " - + " AND LOWER(name) LIKE :nameLike ESCAPE '!' " - + " AND (:includeDeleted = 'ALL' " - + " OR (:includeDeleted = 'DELETED' AND deleted = TRUE) " - + " OR (:includeDeleted = 'NON_DELETED' AND deleted = FALSE)) " - + "ORDER BY name, id LIMIT :limit OFFSET :offset", - connectionType = POSTGRES) - @RegisterRowMapper(ContainerSummaryRowMapper.class) - List listDirectChildSummariesByParentHash( - @Bind("parentHash") String parentHash, - @Bind("parentHashChild") String parentHashChild, - @Bind("nameLike") String nameLike, - @Bind("includeDeleted") String includeDeleted, - @Bind("limit") int limit, - @Bind("offset") int offset); - - @ConnectionAwareSqlQuery( - value = - "SELECT count(fqnHash) FROM storage_container_entity " - + "WHERE fqnHash LIKE :parentHash AND fqnHash NOT LIKE :parentHashChild " - + " AND LOWER(name) LIKE :nameLike ESCAPE '!' " - + " AND (:includeDeleted = 'ALL' " - + " OR (:includeDeleted = 'DELETED' AND deleted = TRUE) " - + " OR (:includeDeleted = 'NON_DELETED' AND deleted = FALSE))", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM storage_container_entity " - + "WHERE fqnHash LIKE :parentHash AND fqnHash NOT LIKE :parentHashChild " - + " AND LOWER(name) LIKE :nameLike ESCAPE '!' " - + " AND (:includeDeleted = 'ALL' " - + " OR (:includeDeleted = 'DELETED' AND deleted = TRUE) " - + " OR (:includeDeleted = 'NON_DELETED' AND deleted = FALSE))", - connectionType = POSTGRES) - int countDirectChildrenByParentHash( - @Bind("parentHash") String parentHash, - @Bind("parentHashChild") String parentHashChild, - @Bind("nameLike") String nameLike, - @Bind("includeDeleted") String includeDeleted); - - /** - * Cascade an FQN rename to every descendant container row when a parent is reassigned - * (#24294). The generic {@link EntityDAO#updateFqn(String, String)} only rewrites the - * top-level {@code $.fullyQualifiedName} via MySQL {@code JSON_REPLACE}, which leaves - * nested column FQNs ({@code $.dataModel.columns[*].fullyQualifiedName}) pointing at the - * old parent — silently breaking column lookups on MySQL. Postgres works only by accident - * because the base impl does a global {@code REPLACE(json::text, ...)}. - * - *

This override rewrites every {@code "fullyQualifiedName": "OLD_PREFIX..."} occurrence - * in the JSON document so column FQNs follow their container. {@code WHERE fqnHash LIKE - * 'oldHash.%'} restricts the update to descendants — the moved row itself updates via the - * standard {@code storeEntity} path after {@code setFullyQualifiedName} runs in memory. - * - *

On SQL interpolation: mirrors the pre-existing {@link EntityDAO#updateFqn} - * pattern — values are spliced into the SQL via {@link String#format} because the - * connection-aware {@code @SqlUpdate} dispatcher takes the full statement as a single - * {@code }/{@code } bind. The values come from server-side - * code (the FQN computed by {@code setFullyQualifiedName}, not user-supplied input), and - * {@link ListFilter#escapeApostrophe} handles the only SQL meta-character that can appear - * in a validated entity name. If a future code path lets arbitrary strings reach this - * method, swap to a parameterised form with {@code @Bind} parameters. - */ - @Override - default void updateFqn(String oldPrefix, String newPrefix) { - if (!getNameHashColumn().equals("fqnHash")) { - return; - } - String oldHash = FullyQualifiedName.buildHash(oldPrefix); - String newHash = FullyQualifiedName.buildHash(newPrefix); - String mySqlUpdate = - String.format( - "UPDATE %s SET json = CAST(REPLACE(CAST(json AS CHAR), " - + "'\"fullyQualifiedName\": \"%s.', '\"fullyQualifiedName\": \"%s.') AS JSON), " - + "fqnHash = REPLACE(fqnHash, '%s.', '%s.') " - + "WHERE fqnHash LIKE '%s.%%'", - getTableName(), - escapeApostrophe(oldPrefix), - escapeApostrophe(newPrefix), - oldHash, - newHash, - oldHash); - String postgresUpdate = - String.format( - "UPDATE %s SET json = REPLACE(json::text, " - + "'\"fullyQualifiedName\": \"%s.', '\"fullyQualifiedName\": \"%s.')::jsonb, " - + "fqnHash = REPLACE(fqnHash, '%s.', '%s.') " - + "WHERE fqnHash LIKE '%s.%%'", - getTableName(), - escapeApostrophe(oldPrefix), - escapeApostrophe(newPrefix), - oldHash, - newHash, - oldHash); - updateFqnInternal(mySqlUpdate, postgresUpdate); - } - - /** - * Cheap descendant count used by the PATCH re-parent guard (#24294) to short-circuit - * absurd subtree moves before any cascade work runs. {@code fqnHash LIKE 'oldHash.%'} - * matches every descendant row (excluding the moved container itself) and the index on - * {@code fqnHash} makes this an O(log n) lookup. - */ - @SqlQuery("SELECT COUNT(*) FROM storage_container_entity WHERE fqnHash LIKE :prefixLike") - int countDescendantsByPrefix(@Bind("prefixLike") String prefixLike); - } - - class ContainerSummaryRowMapper implements RowMapper { - @Override - public Container map(ResultSet rs, StatementContext ctx) throws SQLException { - return new Container() - .withId(UUID.fromString(rs.getString("id"))) - .withName(rs.getString("name")) - .withDisplayName(rs.getString("displayName")) - .withFullyQualifiedName(rs.getString("fqn")) - .withDescription(rs.getString("description")) - .withDeleted(rs.getBoolean("deleted")); - } - } - - interface SearchServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "search_service_entity"; - } - - @Override - default Class getEntityClass() { - return SearchService.class; - } - } - - interface SecurityServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "security_service_entity"; - } - - @Override - default Class getEntityClass() { - return SecurityService.class; - } - } - - interface ApiServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "api_service_entity"; - } - - @Override - default Class getEntityClass() { - return ApiService.class; - } - } - - interface DriveServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "drive_service_entity"; - } - - @Override - default Class getEntityClass() { - return DriveService.class; - } - } - - interface DirectoryDAO extends EntityDAO { - @Override - default String getTableName() { - return "directory_entity"; - } - - @Override - default Class getEntityClass() { - return Directory.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listBefore( - getTableName(), filter.getQueryParams(), sqlCondition, limit, beforeName, beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listAfter( - getTableName(), filter.getQueryParams(), sqlCondition, limit, afterName, afterId); - } - - @Override - default int listCount(ListFilter filter) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listCount(filter); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), sqlCondition); - } - - @SqlQuery( - value = - "SELECT json FROM (" - + "SELECT name,id, ce.json FROM

ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity = 'directory' AND toEntity = 'directory' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + " AND " - + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " - + "ORDER BY name DESC,id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id") - List listBefore( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId); - - @SqlQuery( - value = - "SELECT ce.json FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity = 'directory' AND toEntity = 'directory' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + " AND " - + "(name > :afterName OR (name = :afterName AND id > :afterId)) " - + "ORDER BY name,id " - + "LIMIT :limit") - List listAfter( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId); - - @ConnectionAwareSqlQuery( - value = - "SELECT count() FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity = 'directory' AND toEntity = 'directory' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + "", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity = 'directory' AND toEntity = 'directory' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + "", - connectionType = POSTGRES) - int listCount( - @Define("table") String table, - @Define("nameHashColumn") String nameHashColumn, - @BindMap Map params, - @Define("sqlCondition") String mysqlCond); - } - - interface FileDAO extends EntityDAO { - @Override - default String getTableName() { - return "file_entity"; - } - - @Override - default Class getEntityClass() { - return File.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listBefore( - getTableName(), filter.getQueryParams(), sqlCondition, limit, beforeName, beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listAfter( - getTableName(), filter.getQueryParams(), sqlCondition, limit, afterName, afterId); - } - - @Override - default int listCount(ListFilter filter) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listCount(filter); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), sqlCondition); - } - - @SqlQuery( - value = - "SELECT json FROM (" - + "SELECT name,id, ce.json FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity IN ('directory', 'spreadsheet') AND toEntity = 'file' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + " AND " - + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " - + "ORDER BY name DESC,id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id") - List listBefore( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId); - - @SqlQuery( - value = - "SELECT ce.json FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity IN ('directory', 'spreadsheet') AND toEntity = 'file' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + " AND " - + "(name > :afterName OR (name = :afterName AND id > :afterId)) " - + "ORDER BY name,id " - + "LIMIT :limit") - List listAfter( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId); - - @ConnectionAwareSqlQuery( - value = - "SELECT count() FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity IN ('directory', 'spreadsheet') AND toEntity = 'file' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + "", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity IN ('directory', 'spreadsheet') AND toEntity = 'file' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + "", - connectionType = POSTGRES) - int listCount( - @Define("table") String table, - @Define("nameHashColumn") String nameHashColumn, - @BindMap Map params, - @Define("sqlCondition") String mysqlCond); - } - - interface SpreadsheetDAO extends EntityDAO { - @Override - default String getTableName() { - return "spreadsheet_entity"; - } - - @Override - default Class getEntityClass() { - return Spreadsheet.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listBefore( - getTableName(), filter.getQueryParams(), sqlCondition, limit, beforeName, beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listAfter( - getTableName(), filter.getQueryParams(), sqlCondition, limit, afterName, afterId); - } - - @Override - default int listCount(ListFilter filter) { - boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); - String condition = filter.getCondition(); - if (!root) { - return EntityDAO.super.listCount(filter); - } - String sqlCondition = String.format("%s AND er.toId is NULL", condition); - return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), sqlCondition); - } - - @SqlQuery( - value = - "SELECT json FROM (" - + "SELECT name,id, ce.json FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity = 'directory' AND toEntity = 'spreadsheet' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + " AND " - + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " - + "ORDER BY name DESC,id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id") - List listBefore( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId); - - @SqlQuery( - value = - "SELECT ce.json FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity = 'directory' AND toEntity = 'spreadsheet' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + " AND " - + "(name > :afterName OR (name = :afterName AND id > :afterId)) " - + "ORDER BY name,id " - + "LIMIT :limit") - List listAfter( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId); - - @ConnectionAwareSqlQuery( - value = - "SELECT count() FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity = 'directory' AND toEntity = 'spreadsheet' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + "", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM
ce " - + "LEFT JOIN (" - + " SELECT toId FROM entity_relationship " - + " WHERE fromEntity = 'directory' AND toEntity = 'spreadsheet' AND relation = 0 " - + ") er " - + "on ce.id = er.toId " - + "", - connectionType = POSTGRES) - int listCount( - @Define("table") String table, - @Define("nameHashColumn") String nameHashColumn, - @BindMap Map params, - @Define("sqlCondition") String mysqlCond); - } - - interface WorksheetDAO extends EntityDAO { - @Override - default String getTableName() { - return "worksheet_entity"; - } - - @Override - default Class getEntityClass() { - return Worksheet.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface SearchIndexDAO extends EntityDAO { - @Override - default String getTableName() { - return "search_index_entity"; - } - - @Override - default Class getEntityClass() { - return SearchIndex.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface EntityExtensionDAO { - @ConnectionAwareSqlUpdate( - value = - "REPLACE INTO entity_extension(id, extension, jsonSchema, json) " - + "VALUES (:id, :extension, :jsonSchema, :json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_extension(id, extension, jsonSchema, json) " - + "VALUES (:id, :extension, :jsonSchema, (:json :: jsonb)) " - + "ON CONFLICT (id, extension) DO UPDATE SET jsonSchema = EXCLUDED.jsonSchema, json = EXCLUDED.json", - connectionType = POSTGRES) - void insert( - @BindUUID("id") UUID id, - @Bind("extension") String extension, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json); - - @Transaction - @ConnectionAwareSqlBatch( - value = - "REPLACE INTO entity_extension(id, extension, jsonSchema, json) " - + "VALUES (:id, :extension, :jsonSchema, :json)", - connectionType = MYSQL) - @ConnectionAwareSqlBatch( - value = - "INSERT INTO entity_extension(id, extension, jsonSchema, json) " - + "VALUES (:id, :extension, :jsonSchema, (:json :: jsonb)) " - + "ON CONFLICT (id, extension) DO UPDATE SET jsonSchema = EXCLUDED.jsonSchema, json = EXCLUDED.json", - connectionType = POSTGRES) - void insertMany( - @BindUUID("id") List id, - @Bind("extension") List extension, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") List json); - - @ConnectionAwareSqlUpdate( - value = "UPDATE entity_extension SET json = :json where (json -> '$.id') = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE entity_extension SET json = (:json :: jsonb) where (json ->> 'id) = :id", - connectionType = POSTGRES) - void update(@BindUUID("id") UUID id, @Bind("json") String json); - - @SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension") - String getExtension(@BindUUID("id") UUID id, @Bind("extension") String extension); - - @SqlQuery( - "SELECT id, extension, json " - + "FROM entity_extension " - + "WHERE id IN () AND extension LIKE :extension " - + "ORDER BY id, extension") - @RegisterRowMapper(ExtensionRecordWithIdMapper.class) - List getExtensionsBatch( - @BindList("ids") List ids, - @BindConcat( - value = "extension", - parts = {":extensionPrefix", ".%"}) - String extensionPrefix); - - @SqlQuery( - "SELECT id, extension, json " - + "FROM entity_extension " - + "WHERE id IN () AND extension = :extension " - + "ORDER BY id, extension") - @RegisterRowMapper(ExtensionRecordWithIdMapper.class) - List getExtensionBatch( - @BindList("ids") List ids, @Bind("extension") String extension); - - @SqlQuery( - "SELECT id, extension, json, jsonschema " - + "FROM entity_extension " - + "WHERE extension LIKE :extension " - + "ORDER BY id, extension") - @RegisterRowMapper(ExtensionWithIdAndSchemaRowMapper.class) - List getExtensionsByPrefixBatch( - @BindConcat( - value = "extension", - parts = {":extensionPrefix", "%"}) - String extensionPrefix); - - @Transaction - @ConnectionAwareSqlBatch( - value = - "INSERT INTO entity_extension (id, extension, json, jsonschema) " - + "VALUES (:id, :extension, :json, :jsonschema) " - + "ON DUPLICATE KEY UPDATE json = VALUES(json), jsonschema = VALUES(jsonschema)", - connectionType = MYSQL) - @ConnectionAwareSqlBatch( - value = - "INSERT INTO entity_extension (id, extension, json,jsonschema) VALUES (:id, :extension, :json::jsonb,:jsonschema) " - + "ON CONFLICT (id, extension) DO UPDATE SET json = EXCLUDED.json , jsonschema = EXCLUDED.jsonschema", - connectionType = POSTGRES) - void bulkUpsertExtensions( - @BindBean List extensionWithIdObjects); - - @RegisterRowMapper(ExtensionMapper.class) - @SqlQuery( - "SELECT extension, json FROM entity_extension WHERE id = :id AND extension " - + "LIKE CONCAT (:extensionPrefix, '.%') " - + "ORDER BY extension") - List getExtensions( - @BindUUID("id") UUID id, @Bind("extensionPrefix") String extensionPrefix); - - @RegisterRowMapper(ExtensionMapper.class) - @SqlQuery( - "SELECT extension, json FROM entity_extension WHERE id = :id AND jsonschema = :jsonSchema " - + "ORDER BY extension") - List getExtensionsByJsonSchema( - @BindUUID("id") UUID id, @Bind("jsonSchema") String jsonSchema); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT id, updatedAt, json FROM entity_extension " - + "WHERE updatedAt >= :startTs " - + "AND updatedAt <= :endTs " - + "AND jsonSchema = :entityType " - + "UNION " - + "SELECT id, updatedAt, json FROM
" - + "WHERE updatedAt >= :startTs AND " - + "updatedAt <= :endTs " - + ") combined WHERE 1=1 " - + " " - + "ORDER BY updatedAt DESC, id DESC " - + "LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT id, updatedAt, json FROM entity_extension " - + "WHERE updatedAt >= :startTs " - + "AND updatedAt <= :endTs " - + "AND jsonSchema = :entityType " - + "UNION " - + "SELECT id, updatedAt, json::jsonb FROM
" - + "WHERE updatedAt >= :startTs AND " - + "updatedAt <= :endTs " - + ") combined WHERE 1=1 " - + " " - + "ORDER BY updatedAt DESC, id DESC " - + "LIMIT :limit", - connectionType = POSTGRES) - @RegisterRowMapper(ExtensionMapper.class) - List getEntityHistoryByTimestampRange( - @Define("table") String table, - @Bind("startTs") long startTs, - @Bind("endTs") long endTs, - @Define("cursorCondition") String cursorCondition, - @Bind("entityType") String entityType, - @Bind("cursorUpdatedAt") Long cursorUpdatedAt, - @Bind("cursorId") String cursorId, - @Bind("limit") int limit); - - @SqlQuery( - value = - "SELECT SUM(cnt) FROM (" - + "SELECT COUNT(*) AS cnt FROM entity_extension " - + "WHERE updatedAt >= :startTs " - + "AND updatedAt <= :endTs " - + "AND jsonSchema = :entityType " - + "UNION ALL " - + "SELECT COUNT(*) AS cnt FROM
" - + "WHERE updatedAt >= :startTs AND " - + "updatedAt <= :endTs" - + ") total_counts") - int getEntityHistoryByTimestampRangeCount( - @Define("table") String table, - @Bind("startTs") long startTs, - @Bind("endTs") long endTs, - @Bind("entityType") String entityType); - - @RegisterRowMapper(ExtensionMapper.class) - @SqlQuery( - "SELECT extension, json FROM entity_extension WHERE id = :id AND extension " - + "LIKE CONCAT (:extensionPrefix, '.%') " - + "ORDER BY extension DESC " - + "LIMIT :limit OFFSET :offset") - List getExtensionsWithOffset( - @BindUUID("id") UUID id, - @Bind("extensionPrefix") String extensionPrefix, - @Bind("limit") int limit, - @Bind("offset") int offset); - - @SqlUpdate("DELETE FROM entity_extension WHERE id = :id AND extension = :extension") - void delete(@BindUUID("id") UUID id, @Bind("extension") String extension); - - @SqlUpdate("DELETE FROM entity_extension WHERE extension = :extension") - void deleteExtension(@Bind("extension") String extension); - - @SqlUpdate("DELETE FROM entity_extension WHERE id = :id") - void deleteAll(@BindUUID("id") UUID id); - - @SqlUpdate("DELETE FROM entity_extension WHERE id IN ()") - void deleteAllBatch(@BindList("ids") List ids); - } - - class EntityVersionPair { - @Getter private final Double version; - @Getter private final String entityJson; - - public EntityVersionPair(ExtensionRecord extensionRecord) { - this.version = EntityUtil.getVersion(extensionRecord.extensionName()); - this.entityJson = extensionRecord.extensionJson(); - } - } - - record ExtensionRecord(String extensionName, String extensionJson) {} - - record ExtensionRecordWithId(UUID id, String extensionName, String extensionJson) {} - - class ExtensionMapper implements RowMapper { - @Override - public ExtensionRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new ExtensionRecord(rs.getString("extension"), rs.getString("json")); - } - } - - class ExtensionRecordWithIdMapper implements RowMapper { - @Override - public ExtensionRecordWithId map(ResultSet rs, StatementContext ctx) throws SQLException { - String id = rs.getString("id"); - String extensionName = rs.getString("extension"); - String extensionJson = rs.getString("json"); - return new ExtensionRecordWithId(UUID.fromString(id), extensionName, extensionJson); - } - } - - @Getter - @Setter - @Builder - class ExtensionWithIdAndSchemaObject { - private String id; - private String extension; - private String json; - private String jsonschema; - } - - class ExtensionWithIdAndSchemaRowMapper implements RowMapper { - @Override - public ExtensionWithIdAndSchemaObject map(ResultSet rs, StatementContext ctx) - throws SQLException { - String id = rs.getString("id"); - String extensionName = rs.getString("extension"); - String extensionJson = rs.getString("json"); - String jsonSchema = rs.getString("jsonschema"); - return new ExtensionWithIdAndSchemaObject(id, extensionName, extensionJson, jsonSchema); - } - } - - @Getter - @Builder - class EntityRelationshipRecord { - private UUID id; - private String type; - private String json; - } - - @Getter - @Builder - class EntityRelationshipCount { - private UUID id; - private Integer count; - } - - @Getter - @Builder - class RelationTypeUsageCount { - private String relationType; - private Integer count; - } - - @Getter - @Builder - class EntityRelationshipObject { - private String fromId; - private String toId; - private String fromEntity; - private String toEntity; - private int relation; - private String json; - private String jsonSchema; - } - - @Getter - @Builder - class ReportDataRow { - private String rowNum; - private ReportData reportData; - } - - @Getter - @Builder - class QueryList { - private String fqn; - private Query query; - } - - interface EntityRelationshipDAO { - default void insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation) { - insert(fromId, toId, fromEntity, toEntity, relation, "", null); - } - - default void insert( - UUID fromId, UUID toId, String fromEntity, String toEntity, int relation, String json) { - insert(fromId, toId, fromEntity, toEntity, relation, "", json); - } - - default void bulkInsertToRelationship( - UUID fromId, List toIds, String fromEntity, String toEntity, int relation) { - - List insertToRelationship = - toIds.stream() - .map( - testCase -> - EntityRelationshipObject.builder() - .fromId(fromId.toString()) - .toId(testCase.toString()) - .fromEntity(fromEntity) - .toEntity(toEntity) - .relation(relation) - .build()) - .collect(Collectors.toList()); - - bulkInsertTo(insertToRelationship); - } - - default void bulkRemoveToRelationship( - UUID fromId, List toIds, String fromEntity, String toEntity, int relation) { - - List toIdsAsString = toIds.stream().map(UUID::toString).toList(); - bulkRemoveTo(fromId, toIdsAsString, fromEntity, toEntity, relation); - } - - default void bulkRemoveFromRelationship( - List fromIds, UUID toId, String fromEntity, String toEntity, int relation) { - - List fromIdsAsString = fromIds.stream().map(UUID::toString).toList(); - bulkRemoveFrom(fromIdsAsString, toId, fromEntity, toEntity, relation); - } - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, relationType, json) " - + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation, :relationType, :json) " - + "ON DUPLICATE KEY UPDATE json = :json", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, relationType, json) VALUES " - + "(:fromId, :toId, :fromEntity, :toEntity, :relation, :relationType, (:json :: jsonb)) " - + "ON CONFLICT (fromId, toId, relation, relationType) DO UPDATE SET json = EXCLUDED.json", - connectionType = POSTGRES) - void insert( - @BindUUID("fromId") UUID fromId, - @BindUUID("toId") UUID toId, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation, - @Bind("relationType") String relationType, - @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = - "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES ", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES " - + "ON CONFLICT DO NOTHING", - connectionType = POSTGRES) - void bulkInsertTo( - @BindBeanList( - value = "values", - propertyNames = {"fromId", "toId", "fromEntity", "toEntity", "relation"}) - List values); - - @ConnectionAwareSqlUpdate( - value = - "INSERT IGNORE INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " - + "SELECT :fromId, t.id, :fromEntity, :toEntity, :relation " - + "FROM
t", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " - + "SELECT :fromId, t.id, :fromEntity, :toEntity, :relation " - + "FROM
t " - + "ON CONFLICT DO NOTHING", - connectionType = POSTGRES) - void bulkInsertAllToRelationship( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation, - @Define("table") String table); - - @ConnectionAwareSqlUpdate( - value = - "INSERT IGNORE INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " - + "SELECT :fromId, t.id, :fromEntity, :toEntity, :relation " - + "FROM
t " - + "WHERE t.id NOT IN ()", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " - + "SELECT :fromId, t.id, :fromEntity, :toEntity, :relation " - + "FROM
t " - + "WHERE t.id NOT IN () " - + "ON CONFLICT DO NOTHING", - connectionType = POSTGRES) - void bulkInsertAllToRelationshipWithExclusions( - @BindList("exclusionIds") List excludedIds, - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation, - @Define("table") String table); - - @SqlUpdate( - value = - "DELETE FROM entity_relationship WHERE fromId = :fromId " - + "AND fromEntity = :fromEntity AND toId IN () " - + "AND toEntity = :toEntity AND relation = :relation") - void bulkRemoveTo( - @BindUUID("fromId") UUID fromId, - @BindList("toIds") List toIds, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - @SqlUpdate( - "DELETE FROM entity_relationship " - + "WHERE fromEntity = :fromEntity " - + "AND fromId IN () " - + "AND toEntity = :toEntity " - + "AND relation = :relation " - + "AND toId = :toId") - void bulkRemoveFrom( - @BindList("fromIds") List fromIds, - @BindUUID("toId") UUID toId, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - @SqlUpdate( - "UPDATE entity_relationship " - + "SET fromId = :newFromId " - + "WHERE fromId = :oldFromId " - + "AND fromEntity = :fromEntity " - + "AND toEntity = :toEntity " - + "AND relation = :relation " - + "AND toId IN ()") - void bulkUpdateFromId( - @BindUUID("oldFromId") UUID oldFromId, - @BindUUID("newFromId") UUID newFromId, - @BindList("toIds") List toIds, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - // - // Find to operations - // - @SqlQuery( - "SELECT toId, toEntity, json FROM entity_relationship " - + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation IN ()") - @RegisterRowMapper(ToRelationshipMapper.class) - List findTo( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @BindList("relation") List relation); - - @SqlQuery( - "SELECT * FROM entity_relationship er1 JOIN entity_relationship er2 ON er1.toId = er2.toId WHERE er1.relation = 10 AND er1.fromEntity = 'domain' AND er2.fromId = :fromId AND er2.fromEntity = :fromEntity AND er2.relation = 13") - @RegisterRowMapper(RelationshipObjectMapper.class) - List findDownstreamDomains( - @BindUUID("fromId") UUID fromId, @Bind("fromEntity") String fromEntity); - - @SqlQuery( - "SELECT * FROM entity_relationship er1 JOIN entity_relationship er2 ON er1.toId = er2.fromId WHERE er1.relation = 10 AND er1.fromEntity = 'domain' AND er2.toId = :toId AND er2.toEntity = :toEntity AND er2.relation = 13") - @RegisterRowMapper(RelationshipObjectMapper.class) - List findUpstreamDomains( - @BindUUID("toId") UUID toId, @Bind("toEntity") String toEntity); - - @SqlQuery( - "select count(*) from entity_relationship where fromId in (select toId from entity_relationship where fromId = :fromDomainId and fromEntity = 'domain' and relation = 10) AND toId in (select toId from entity_relationship where fromId = :toDomainId and fromEntity = 'domain' and relation = 10) and relation = 13") - Integer countDomainChildAssets( - @BindUUID("fromDomainId") UUID fromDomainId, @BindUUID("toDomainId") UUID toId); - - @SqlQuery( - "SELECT * FROM entity_relationship er1 JOIN entity_relationship er2 ON er1.toId = er2.toId WHERE er1.relation = 10 AND er1.fromEntity = 'dataProduct' AND er2.fromId = :fromId AND er2.fromEntity = :fromEntity AND er2.relation = 13") - @RegisterRowMapper(RelationshipObjectMapper.class) - List findDownstreamDataProducts( - @BindUUID("fromId") UUID fromId, @Bind("fromEntity") String fromEntity); - - @SqlQuery( - "SELECT * FROM entity_relationship er1 JOIN entity_relationship er2 ON er1.toId = er2.fromId WHERE er1.relation = 10 AND er1.fromEntity = 'dataProduct' AND er2.toId = :toId AND er2.toEntity = :toEntity AND er2.relation = 13") - @RegisterRowMapper(RelationshipObjectMapper.class) - List findUpstreamDataProducts( - @BindUUID("toId") UUID toId, @Bind("toEntity") String toEntity); - - @SqlQuery( - "select count(*) from entity_relationship where fromId in (select toId from entity_relationship where fromId = :fromDataProductId and fromEntity = 'dataProduct' and relation = 10) AND toId in (select toId from entity_relationship where fromId = :toDataProductId and fromEntity = 'dataProduct' and relation = 10) and relation = 13") - Integer countDataProductsChildAssets( - @BindUUID("fromDataProductId") UUID fromDataProductId, - @BindUUID("toDataProductId") UUID toDataProductId); - - default List findTo(UUID fromId, String fromEntity, int relation) { - return findTo(fromId, fromEntity, List.of(relation)); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE fromId IN () " - + "AND relation = :relation " - + "AND fromEntity = :fromEntityType " - + "AND toEntity = :toEntityType " - + "AND deleted = FALSE") - @UseRowMapper(RelationshipObjectMapper.class) - List findToBatch( - @BindList("fromIds") List fromIds, - @Bind("relation") int relation, - @Bind("fromEntityType") String fromEntityType, - @Bind("toEntityType") String toEntityType); - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE fromId IN () " - + "AND relation = :relation " - + "AND toEntity = :toEntityType " - + "AND deleted = FALSE") - @UseRowMapper(RelationshipObjectMapper.class) - List findToBatch( - @BindList("fromIds") List fromIds, - @Bind("relation") int relation, - @Bind("toEntityType") String toEntityType); - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE fromId IN () " - + "AND relation = :relation " - + "AND toEntity = :toEntityType " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findToBatchWithCondition( - @BindList("fromIds") List fromIds, - @Bind("relation") int relation, - @Bind("toEntityType") String toEntityType, - @Define("cond") String condition); - - default List findToBatch( - List fromIds, int relation, String toEntityType, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findToBatchWithCondition(fromIds, relation, toEntityType, condition); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE fromId IN () " - + "AND relation = :relation " - + "AND fromEntity = :fromEntityType " - + "AND toEntity = :toEntityType " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findToBatchWithCondition( - @BindList("fromIds") List fromIds, - @Bind("relation") int relation, - @Bind("fromEntityType") String fromEntityType, - @Bind("toEntityType") String toEntityType, - @Define("cond") String condition); - - default List findToBatch( - List fromIds, - String fromEntityType, - String toEntityType, - int relation, - Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findToBatchWithCondition(fromIds, relation, fromEntityType, toEntityType, condition); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE fromId IN () " - + "AND relation = :relation " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findToBatchAllTypesWithCondition( - @BindList("fromIds") List fromIds, - @Bind("relation") int relation, - @Define("cond") String condition); - - default List findToBatchAllTypes( - List fromIds, int relation, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findToBatchAllTypesWithCondition(fromIds, relation, condition); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE fromId IN () " - + "AND relation IN () " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findToBatchAllTypesWithRelationsCondition( - @BindList("fromIds") List fromIds, - @BindList("relations") List relations, - @Define("cond") String condition); - - default List findToBatchAllTypes( - List fromIds, List relations, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findToBatchAllTypesWithRelationsCondition(fromIds, relations, condition); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE fromId IN () " - + "AND fromEntity = :fromEntity " - + "AND relation IN () " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findToBatchWithRelationsAndCondition( - @BindList("fromIds") List fromIds, - @Bind("fromEntity") String fromEntity, - @BindList("relations") List relations, - @Define("cond") String condition); - - default List findToBatchWithRelations( - List fromIds, String fromEntity, List relations, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findToBatchWithRelationsAndCondition(fromIds, fromEntity, relations, condition); - } - - default List findToBatchWithRelations( - List fromIds, String fromEntity, List relations) { - return findToBatchWithRelations(fromIds, fromEntity, relations, Include.ALL); - } - - @SqlQuery( - "SELECT toId, toEntity, json FROM entity_relationship " - + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity") - @RegisterRowMapper(ToRelationshipMapper.class) - List findTo( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @Bind("relation") int relation, - @Bind("toEntity") String toEntity); - - @SqlQuery( - "SELECT toId FROM entity_relationship " - + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity") - @RegisterRowMapper(ToRelationshipMapper.class) - List findToIds( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @Bind("relation") int relation, - @Bind("toEntity") String toEntity); - - @SqlQuery( - "SELECT COUNT(*) FROM entity_relationship " - + "WHERE fromId = :fromId AND toId = :toId AND fromEntity = :fromEntity AND toEntity = :toEntity AND relation = :relation") - int existsRelationship( - @BindUUID("fromId") UUID fromId, - @BindUUID("toId") UUID toId, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT fromId, COUNT(toId) FROM entity_relationship " - + "WHERE fromId IN () AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity " - + "GROUP BY fromId") - @RegisterRowMapper(ToRelationshipCountMapper.class) - List countFindTo( - @BindList("fromIds") List fromIds, - @Bind("fromEntity") String fromEntity, - @Bind("relation") int relation, - @Bind("toEntity") String toEntity); - - @SqlQuery( - "SELECT toId, toEntity, json FROM entity_relationship " - + "WHERE fromEntity = :fromEntity AND toEntity = :toEntity AND relation = :relation") - @RegisterRowMapper(ToRelationshipMapper.class) - List findAllByEntityTypes( - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT CASE WHEN relationType = '' THEN 'relatedTo' ELSE relationType END AS relationType, " - + "COUNT(*) AS cnt FROM entity_relationship " - + "WHERE fromEntity = :fromEntity AND toEntity = :toEntity AND relation = :relation " - + "GROUP BY CASE WHEN relationType = '' THEN 'relatedTo' ELSE relationType END") - @RegisterRowMapper(RelationTypeUsageCountMapper.class) - List countByRelationType( - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT COUNT(toId) FROM entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " - + "AND relation IN ()") - @RegisterRowMapper(ToRelationshipMapper.class) - int countFindTo( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @BindList("relation") List relation); - - @SqlQuery( - "SELECT toId, toEntity, json FROM entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " - + "AND relation IN () ORDER BY toId LIMIT :limit OFFSET :offset") - @RegisterRowMapper(ToRelationshipMapper.class) - List findToWithOffset( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @BindList("relation") List relation, - @Bind("offset") int offset, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT toId, toEntity, json FROM entity_relationship " - + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) =:fromId OR fromId = :fromId AND relation = :relation " - + "ORDER BY toId", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT toId, toEntity, json FROM entity_relationship " - + "WHERE json->'pipeline'->>'id' =:fromId OR fromId = :fromId AND relation = :relation " - + "ORDER BY toId", - connectionType = POSTGRES) - @RegisterRowMapper(ToRelationshipMapper.class) - List findToPipeline( - @BindUUID("fromId") UUID fromId, @Bind("relation") int relation); - - // - // Find from operations - // - @SqlQuery( - "SELECT fromId, fromEntity, json FROM entity_relationship " - + "WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation AND fromEntity = :fromEntity ") - @RegisterRowMapper(FromRelationshipMapper.class) - List findFrom( - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation, - @Bind("fromEntity") String fromEntity); - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE toId IN () " - + "AND relation = :relation " - + "AND deleted = FALSE") - @UseRowMapper(RelationshipObjectMapper.class) - List findFromBatch( - @BindList("toIds") List toIds, @Bind("relation") int relation); - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE toId IN () " - + "AND relation = :relation " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findFromBatchWithCondition( - @BindList("toIds") List toIds, - @Bind("relation") int relation, - @Define("cond") String condition); - - default List findFromBatch( - List toIds, int relation, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findFromBatchWithCondition(toIds, relation, condition); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE toId IN () " - + "AND relation = :relation " - + "AND fromEntity = :fromEntityType " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findFromBatchWithEntityTypeAndCondition( - @BindList("toIds") List toIds, - @Bind("relation") int relation, - @Bind("fromEntityType") String fromEntityType, - @Define("cond") String condition); - - default List findFromBatch( - List toIds, int relation, String fromEntityType, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findFromBatchWithEntityTypeAndCondition(toIds, relation, fromEntityType, condition); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE toId IN () " - + "AND toEntity = :toEntityType " - + "AND relation IN () " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findFromBatchWithRelationsAndCondition( - @BindList("toIds") List toIds, - @Bind("toEntityType") String toEntityType, - @BindList("relations") List relations, - @Define("cond") String condition); - - default List findFromBatchWithRelations( - List toIds, String toEntityType, List relations, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findFromBatchWithRelationsAndCondition(toIds, toEntityType, relations, condition); - } - - default List findFromBatchWithRelations( - List toIds, String toEntityType, List relations) { - return findFromBatchWithRelations(toIds, toEntityType, relations, Include.ALL); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE toId IN () " - + "AND relation = :relation " - + "AND fromEntity = :fromEntityType " - + "AND deleted = FALSE") - @UseRowMapper(RelationshipObjectMapper.class) - List findFromBatch( - @BindList("toIds") List toIds, - @Bind("relation") int relation, - @Bind("fromEntityType") String fromEntityType); - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE toId IN () " - + "AND relation = :relation " - + "AND toEntity = :toEntityType " - + "AND deleted = FALSE") - @UseRowMapper(RelationshipObjectMapper.class) - List findFromBatch( - @BindList("toIds") List toIds, - @Bind("toEntityType") String toEntityType, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT fromId, fromEntity, json FROM entity_relationship " - + "WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation") - @RegisterRowMapper(FromRelationshipMapper.class) - List findFrom( - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - // Fetch relationships for specific relation types (TO direction: others -> entity) - // Used for owners, followers, domains, dataProducts, reviewers - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE toId = :entityId AND toEntity = :entityType " - + "AND relation IN () " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findToRelationshipsForEntityWithCondition( - @BindUUID("entityId") UUID entityId, - @Bind("entityType") String entityType, - @BindList("relations") List relations, - @Define("cond") String condition); - - default List findToRelationshipsForEntity( - UUID entityId, String entityType, List relations, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findToRelationshipsForEntityWithCondition(entityId, entityType, relations, condition); - } - - default List findToRelationshipsForEntity( - UUID entityId, String entityType, List relations) { - return findToRelationshipsForEntity(entityId, entityType, relations, Include.ALL); - } - - // Fetch relationships for specific relation types (FROM direction: entity -> others) - // Used for children, experts - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE fromId = :entityId AND fromEntity = :entityType " - + "AND relation IN () " - + "") - @UseRowMapper(RelationshipObjectMapper.class) - List findFromRelationshipsForEntityWithCondition( - @BindUUID("entityId") UUID entityId, - @Bind("entityType") String entityType, - @BindList("relations") List relations, - @Define("cond") String condition); - - default List findFromRelationshipsForEntity( - UUID entityId, String entityType, List relations, Include include) { - String condition = ""; - if (include == null || include == Include.NON_DELETED) { - condition = "AND deleted = FALSE"; - } else if (include == Include.DELETED) { - condition = "AND deleted = TRUE"; - } - return findFromRelationshipsForEntityWithCondition( - entityId, entityType, relations, condition); - } - - default List findFromRelationshipsForEntity( - UUID entityId, String entityType, List relations) { - return findFromRelationshipsForEntity(entityId, entityType, relations, Include.ALL); - } - - @SqlQuery( - "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " - + "FROM entity_relationship " - + "WHERE toId IN () " - + "AND relation = :relation " - + "AND fromEntity = :fromEntityType " - + "AND toEntity = :toEntityType " - + "AND deleted = FALSE") - @UseRowMapper(RelationshipObjectMapper.class) - List findFromBatch( - @BindList("toIds") List toIds, - @Bind("relation") int relation, - @Bind("fromEntityType") String fromEntityType, - @Bind("toEntityType") String toEntityType); - - @ConnectionAwareSqlQuery( - value = - "SELECT fromId, fromEntity, json FROM entity_relationship " - + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) = :toId OR toId = :toId AND relation = :relation " - + "ORDER BY fromId", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT fromId, fromEntity, json FROM entity_relationship " - + "WHERE json->'pipeline'->>'id' = :toId OR toId = :toId AND relation = :relation " - + "ORDER BY fromId", - connectionType = POSTGRES) - @RegisterRowMapper(FromRelationshipMapper.class) - List findFromPipeline( - @BindUUID("toId") UUID toId, @Bind("relation") int relation); - - @ConnectionAwareSqlQuery( - value = - "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship " - + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.source')) = :source AND (toId = :toId AND toEntity = :toEntity) " - + "AND relation = :relation ORDER BY fromId", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship " - + "WHERE json->>'source' = :source AND (toId = :toId AND toEntity = :toEntity) " - + "AND relation = :relation ORDER BY fromId", - connectionType = POSTGRES) - @RegisterRowMapper(RelationshipObjectMapper.class) - List findLineageBySource( - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("source") String source, - @Bind("relation") int relation); - - @ConnectionAwareSqlQuery( - value = - "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship " - + "WHERE (JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) =:toId OR toId = :toId) AND relation = :relation " - + "AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.source')) = :source ORDER BY toId", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship " - + "WHERE (json->'pipeline'->>'id' =:toId OR toId = :toId) AND relation = :relation " - + "AND json->>'source' = :source ORDER BY toId", - connectionType = POSTGRES) - @RegisterRowMapper(RelationshipObjectMapper.class) - List findLineageBySourcePipeline( - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("source") String source, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT count(*) FROM entity_relationship WHERE fromEntity = :fromEntity AND toEntity = :toEntity") - int findIfAnyRelationExist( - @Bind("fromEntity") String fromEntity, @Bind("toEntity") String toEntity); - - @SqlQuery( - "SELECT json FROM entity_relationship WHERE fromId = :fromId " - + " AND toId = :toId " - + " AND relation = :relation ") - String getRelation( - @BindUUID("fromId") UUID fromId, - @BindUUID("toId") UUID toId, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship WHERE fromId = :fromId " - + " AND toId = :toId " - + " AND relation = :relation ") - @RegisterRowMapper(RelationshipObjectMapper.class) - EntityRelationshipObject getRecord( - @BindUUID("fromId") UUID fromId, - @BindUUID("toId") UUID toId, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship where relation = :relation ORDER BY fromId, toId LIMIT :limit OFFSET :offset") - @RegisterRowMapper(RelationshipObjectMapper.class) - List getRecordWithOffset( - @Bind("relation") int relation, @Bind("offset") long offset, @Bind("limit") int limit); - - @SqlQuery( - "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship ORDER BY fromId, toId LIMIT :limit OFFSET :offset") - @RegisterRowMapper(RelationshipObjectMapper.class) - List getAllRelationshipsPaginated( - @Bind("offset") long offset, @Bind("limit") int limit); - - @SqlQuery("SELECT COUNT(*) FROM entity_relationship") - long getTotalRelationshipCount(); - - // - // Delete Operations - // - @SqlUpdate( - "DELETE from entity_relationship WHERE fromId = :fromId " - + "AND fromEntity = :fromEntity AND toId = :toId AND toEntity = :toEntity " - + "AND relation = :relation") - int delete( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - @SqlUpdate( - "DELETE FROM entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " - + "AND toId = :toId AND toEntity = :toEntity AND relation = :relation " - + "AND relationType = :relationType") - int deleteWithRelationType( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation, - @Bind("relationType") String relationType); - - // Delete all the entity relationship fromID --- relation --> entity of type toEntity - @SqlUpdate( - "DELETE from entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " - + "AND relation = :relation AND toEntity = :toEntity") - void deleteFrom( - @BindUUID("fromId") UUID fromId, - @Bind("fromEntity") String fromEntity, - @Bind("relation") int relation, - @Bind("toEntity") String toEntity); - - // Delete all the entity relationship toId <-- relation -- entity of type fromEntity - @SqlUpdate( - "DELETE from entity_relationship WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation " - + "AND fromEntity = :fromEntity") - void deleteTo( - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation, - @Bind("fromEntity") String fromEntity); - - @SqlUpdate( - "DELETE from entity_relationship WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation") - void deleteTo( - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - @SqlUpdate( - "DELETE FROM entity_relationship WHERE toId IN () " - + "AND toEntity = :toEntity AND relation = :relation AND fromEntity = :fromEntity") - void deleteToMany( - @BindList("toIds") List toIds, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation, - @Bind("fromEntity") String fromEntity); - - @SqlUpdate( - "DELETE FROM entity_relationship WHERE toId IN () " - + "AND toEntity = :toEntity AND relation = :relation") - void deleteToMany( - @BindList("toIds") List toIds, - @Bind("toEntity") String toEntity, - @Bind("relation") int relation); - - @SqlUpdate( - "DELETE FROM entity_relationship WHERE fromId IN () " - + "AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity") - void deleteFromMany( - @BindList("fromIds") List fromIds, - @Bind("fromEntity") String fromEntity, - @Bind("relation") int relation, - @Bind("toEntity") String toEntity); - - @SqlUpdate( - "DELETE FROM entity_relationship WHERE fromId IN () " - + "AND fromEntity = :fromEntity AND relation = :relation") - void deleteFromMany( - @BindList("fromIds") List fromIds, - @Bind("fromEntity") String fromEntity, - @Bind("relation") int relation); - - // Optimized deleteAll implementation that splits OR query for better performance - @Transaction - default void deleteAll(UUID id, String entity) { - // Split OR query into two separate deletes for better index usage - deleteAllFrom(id, entity); - deleteAllTo(id, entity); - } - - @SqlUpdate("DELETE FROM entity_relationship WHERE fromId = :id AND fromEntity = :entity") - void deleteAllFrom(@BindUUID("id") UUID id, @Bind("entity") String entity); - - @SqlUpdate("DELETE FROM entity_relationship WHERE toId = :id AND toEntity = :entity") - void deleteAllTo(@BindUUID("id") UUID id, @Bind("entity") String entity); - - // Batch deletion methods for improved performance - @Transaction - default void batchDeleteRelationships(List entityIds, String entityType) { - if (entityIds == null || entityIds.isEmpty()) { - return; - } - - // Process in chunks of 500 to avoid hitting database query limits - int batchSize = 500; - for (int i = 0; i < entityIds.size(); i += batchSize) { - int endIndex = Math.min(i + batchSize, entityIds.size()); - List batch = - entityIds.subList(i, endIndex).stream() - .map(UUID::toString) - .collect(Collectors.toList()); - - batchDeleteFrom(batch, entityType); - batchDeleteTo(batch, entityType); - } - } - - @SqlUpdate( - "DELETE FROM entity_relationship WHERE fromId IN () AND fromEntity = :entityType") - void batchDeleteFrom(@BindList("ids") List ids, @Bind("entityType") String entityType); - - @SqlUpdate("DELETE FROM entity_relationship WHERE toId IN () AND toEntity = :entityType") - void batchDeleteTo(@BindList("ids") List ids, @Bind("entityType") String entityType); - - @SqlUpdate( - "DELETE FROM entity_relationship " - + "WHERE (toId IN () AND toEntity = :entity) " - + " OR (fromId IN () AND fromEntity = :entity)") - void deleteAllByThreadIds(@BindList("ids") List ids, @Bind("entity") String entity); - - @SqlUpdate("DELETE from entity_relationship WHERE fromId = :id or toId = :id") - void deleteAllWithId(@BindUUID("id") UUID id); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM entity_relationship " - + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.source')) = :source AND toId = :toId AND toEntity = :toEntity " - + "AND relation = :relation", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM entity_relationship " - + "WHERE json->>'source' = :source AND (toId = :toId AND toEntity = :toEntity) " - + "AND relation = :relation", - connectionType = POSTGRES) - void deleteLineageBySource( - @BindUUID("toId") UUID toId, - @Bind("toEntity") String toEntity, - @Bind("source") String source, - @Bind("relation") int relation); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM entity_relationship " - + "WHERE (JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) =:toId OR toId = :toId) AND relation = :relation " - + "AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.source')) = :source ORDER BY toId", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM entity_relationship " - + "WHERE (json->'pipeline'->>'id' =:toId OR toId = :toId) AND relation = :relation " - + "AND json->>'source' = :source", - connectionType = POSTGRES) - void deleteLineageBySourcePipeline( - @BindUUID("toId") UUID toId, @Bind("source") String source, @Bind("relation") int relation); - - class FromRelationshipMapper implements RowMapper { - @Override - public EntityRelationshipRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return EntityRelationshipRecord.builder() - .id(UUID.fromString(rs.getString("fromId"))) - .type(rs.getString("fromEntity")) - .json(rs.getString("json")) - .build(); - } - } - - class ToRelationshipMapper implements RowMapper { - @Override - public EntityRelationshipRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return EntityRelationshipRecord.builder() - .id(UUID.fromString(rs.getString("toId"))) - .type(rs.getString("toEntity")) - .json(rs.getString("json")) - .build(); - } - } - - class ToRelationshipCountMapper implements RowMapper { - @Override - public EntityRelationshipCount map(ResultSet rs, StatementContext ctx) throws SQLException { - return EntityRelationshipCount.builder() - .id(UUID.fromString(rs.getString(1))) - .count(rs.getInt(2)) - .build(); - } - } - - class RelationTypeUsageCountMapper implements RowMapper { - @Override - public RelationTypeUsageCount map(ResultSet rs, StatementContext ctx) throws SQLException { - return RelationTypeUsageCount.builder() - .relationType(rs.getString("relationType")) - .count(rs.getInt("cnt")) - .build(); - } - } - - class RelationshipObjectMapper implements RowMapper { - @Override - public EntityRelationshipObject map(ResultSet rs, StatementContext ctx) throws SQLException { - return EntityRelationshipObject.builder() - .fromId(rs.getString("fromId")) - .fromEntity(rs.getString("fromEntity")) - .toEntity(rs.getString("toEntity")) - .toId(rs.getString("toId")) - .relation(rs.getInt("relation")) - .json(rs.getString("json")) - .jsonSchema(rs.getString("jsonSchema")) - .build(); - } - } - } - - interface FeedDAO { - @ConnectionAwareSqlUpdate( - value = "INSERT INTO (json) VALUES (:json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "INSERT INTO (json) VALUES (:json :: jsonb)", - connectionType = POSTGRES) - void insert(@Define("tableName") String tableName, @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = "INSERT INTO thread_entity(json) VALUES (:json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "INSERT INTO thread_entity(json) VALUES (:json :: jsonb)", - connectionType = POSTGRES) - void insert(@Bind("json") String json); - - @SqlQuery("SELECT json FROM WHERE id = :id") - String findById(@Define("tableName") String tableName, @BindUUID("id") UUID id); - - @SqlQuery("SELECT json FROM thread_entity WHERE id = :id") - String findById(@BindUUID("id") UUID id); - - @SqlQuery("SELECT json FROM ORDER BY createdAt DESC") - List list(@Define("tableName") String tableName); - - @SqlQuery("SELECT json FROM thread_entity ORDER BY createdAt DESC") - List list(); - - @SqlQuery("SELECT count(id) FROM ") - int listCount( - @Define("tableName") String tableName, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery("SELECT count(id) FROM thread_entity ") - int listCount(@Define("condition") String condition, @BindMap Map params); - - @SqlUpdate("DELETE FROM WHERE id = :id") - void delete(@Define("tableName") String tableName, @BindUUID("id") UUID id); - - @SqlUpdate("DELETE FROM thread_entity WHERE id = :id") - void delete(@BindUUID("id") UUID id); - - @SqlUpdate("DELETE FROM WHERE id IN ()") - int deleteByIds(@Define("tableName") String tableName, @BindList("ids") List ids); - - @SqlUpdate("DELETE FROM thread_entity WHERE id IN ()") - int deleteByIds(@BindList("ids") List ids); - - @ConnectionAwareSqlUpdate( - value = "UPDATE task_sequence SET id=LAST_INSERT_ID(id+1)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE task_sequence SET id=(id+1) RETURNING id", - connectionType = POSTGRES) - void updateTaskId(); - - @ConnectionAwareSqlQuery(value = "SELECT LAST_INSERT_ID()", connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT id FROM task_sequence LIMIT 1", - connectionType = POSTGRES) - int getTaskId(); - - @SqlQuery("SELECT json FROM WHERE taskId = :id") - String findByTaskId(@Define("tableName") String tableName, @Bind("id") int id); - - @SqlQuery("SELECT json FROM thread_entity WHERE taskId = :id") - String findByTaskId(@Bind("id") int id); - - @SqlQuery("SELECT json FROM ORDER BY createdAt DESC LIMIT :limit") - List list( - @Define("tableName") String tableName, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery("SELECT json FROM thread_entity ORDER BY createdAt DESC LIMIT :limit") - List list( - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM " - + "WHERE type='Announcement' AND (:threadId IS NULL OR id != :threadId) " - + "AND entityId = :entityId " - + "AND (( :startTs >= announcementStart AND :startTs < announcementEnd) " - + "OR (:endTs > announcementStart AND :endTs < announcementEnd) " - + "OR (:startTs <= announcementStart AND :endTs >= announcementEnd))") - List listAnnouncementBetween( - @Define("tableName") String tableName, - @BindUUID("threadId") UUID threadId, - @BindUUID("entityId") UUID entityId, - @Bind("startTs") long startTs, - @Bind("endTs") long endTs); - - @SqlQuery( - "SELECT json FROM thread_entity " - + "WHERE type='Announcement' AND (:threadId IS NULL OR id != :threadId) " - + "AND entityId = :entityId " - + "AND (( :startTs >= announcementStart AND :startTs < announcementEnd) " - + "OR (:endTs > announcementStart AND :endTs < announcementEnd) " - + "OR (:startTs <= announcementStart AND :endTs >= announcementEnd))") - List listAnnouncementBetween( - @BindUUID("threadId") UUID threadId, - @BindUUID("entityId") UUID entityId, - @Bind("startTs") long startTs, - @Bind("endTs") long endTs); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM AND " - + "to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit", - connectionType = POSTGRES) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM AND " - + "MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit", - connectionType = MYSQL) - List listTasksAssigned( - @Define("tableName") String tableName, - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM thread_entity AND " - + "to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit", - connectionType = POSTGRES) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM thread_entity AND " - + "MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit", - connectionType = MYSQL) - List listTasksAssigned( - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM AND " - + "to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) ", - connectionType = POSTGRES) - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM AND " - + "MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) ", - connectionType = MYSQL) - int listCountTasksAssignedTo( - @Define("tableName") String tableName, - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Define("condition") String condition, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM thread_entity AND " - + "to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) ", - connectionType = POSTGRES) - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM thread_entity AND " - + "MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) ", - connectionType = MYSQL) - int listCountTasksAssignedTo( - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Define("condition") String condition, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM " - + "AND (to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) OR createdBy = :username) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit", - connectionType = POSTGRES) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM " - + "AND (MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) OR createdBy = :username) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit", - connectionType = MYSQL) - List listTasksOfUser( - @Define("tableName") String tableName, - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Bind("username") String username, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM thread_entity " - + "AND (to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) OR createdBy = :username) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit", - connectionType = POSTGRES) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM thread_entity " - + "AND (MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) OR createdBy = :username) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit", - connectionType = MYSQL) - List listTasksOfUser( - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Bind("username") String username, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT id FROM WHERE type = 'Conversation' AND createdAt < :cutoffMillis LIMIT :batchSize") - List fetchConversationThreadIdsOlderThan( - @Define("tableName") String tableName, - @Bind("cutoffMillis") long cutoffMillis, - @Bind("batchSize") int batchSize); - - @SqlQuery( - "SELECT id FROM thread_entity WHERE type = 'Conversation' AND createdAt < :cutoffMillis LIMIT :batchSize") - List fetchConversationThreadIdsOlderThan( - @Bind("cutoffMillis") long cutoffMillis, @Bind("batchSize") int batchSize); - - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM " - + "AND (to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) OR createdBy = :username) ", - connectionType = POSTGRES) - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM " - + "AND (MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) OR createdBy = :username) ", - connectionType = MYSQL) - int listCountTasksOfUser( - @Define("tableName") String tableName, - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Bind("username") String username, - @Define("condition") String condition, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM thread_entity " - + "AND (to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) OR createdBy = :username) ", - connectionType = POSTGRES) - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM thread_entity " - + "AND (MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) OR createdBy = :username) ", - connectionType = MYSQL) - int listCountTasksOfUser( - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Bind("username") String username, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM AND createdBy = :username ORDER BY createdAt DESC LIMIT :limit") - List listTasksAssignedByUser( - @Define("tableName") String tableName, - @Bind("username") String username, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM thread_entity AND createdBy = :username ORDER BY createdAt DESC LIMIT :limit") - List listTasksAssigned( - @Bind("username") String username, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery("SELECT count(id) FROM AND createdBy = :username") - int listCountTasksAssignedBy( - @Define("tableName") String tableName, - @Bind("username") String username, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery("SELECT count(id) FROM thread_entity AND createdBy = :username") - int listCountTasksAssignedBy( - @Bind("username") String username, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM thread_entity where type = 'Task' LIMIT :limit OFFSET :paginationOffset") - List listTaskThreadWithOffset( - @Bind("limit") int limit, @Bind("paginationOffset") int paginationOffset); - - @SqlQuery( - "SELECT json FROM thread_entity where type != 'Task' AND createdAt > :cutoffMillis ORDER BY createdAt LIMIT :limit OFFSET :paginationOffset") - List listOtherConversationThreadWithOffset( - @Bind("cutoffMillis") long cutoffMillis, - @Bind("limit") int limit, - @Bind("paginationOffset") int paginationOffset); - - @SqlQuery( - "SELECT json FROM AND " - // Entity for which the thread is about is owned by the user or his teams - + "(entityId in (SELECT toId FROM entity_relationship WHERE " - + "((fromEntity='user' AND fromId= :userId) OR " - + "(fromEntity='team' AND fromId IN ())) AND relation=8) OR " - + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2)))) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByOwner( - @Define("tableName") String tableName, - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM thread_entity AND " - // Entity for which the thread is about is owned by the user or his teams - + "(entityId in (SELECT toId FROM entity_relationship WHERE " - + "((fromEntity='user' AND fromId= :userId) OR " - + "(fromEntity='team' AND fromId IN ())) AND relation=8) OR " - + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2)))) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByOwner( - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT count(id) FROM AND " - + "(entityId in (SELECT toId FROM entity_relationship WHERE " - + "((fromEntity='user' AND fromId= :userId) OR " - + "(fromEntity='team' AND fromId IN ())) AND relation=8) OR " - + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2)))) ") - int listCountThreadsByOwner( - @Define("tableName") String tableName, - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT count(id) FROM thread_entity AND " - + "(entityId in (SELECT toId FROM entity_relationship WHERE " - + "((fromEntity='user' AND fromId= :userId) OR " - + "(fromEntity='team' AND fromId IN ())) AND relation=8) OR " - + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2)))) ") - int listCountThreadsByOwner( - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - value = - "SELECT json " - + " FROM " - + " WHERE testCaseResolutionStatusId = :testCaseResolutionStatusId") - String fetchThreadByTestCaseResolutionStatusId( - @Define("tableName") String tableName, - @BindUUID("testCaseResolutionStatusId") UUID testCaseResolutionStatusId); - - @SqlQuery( - value = - "SELECT json " - + " FROM thread_entity " - + " WHERE testCaseResolutionStatusId = :testCaseResolutionStatusId") - String fetchThreadByTestCaseResolutionStatusId( - @BindUUID("testCaseResolutionStatusId") UUID testCaseResolutionStatusId); - - default List listThreadsByEntityLink( - String tableName, - FeedFilter filter, - EntityLink entityLink, - int limit, - int relation, - String userName, - List teamNames) { - int filterRelation = -1; - if (userName != null && filter.getFilterType() == FilterType.MENTIONS) { - filterRelation = MENTIONED_IN.ordinal(); - } - return listThreadsByEntityLink( - tableName, - entityLink.getFullyQualifiedFieldValue(), - entityLink.getFullyQualifiedFieldType(), - limit, - relation, - userName, - teamNames, - filterRelation, - filter.getCondition(), - filter.getQueryParams()); - } - - default List listThreadsByEntityLink( - FeedFilter filter, - EntityLink entityLink, - int limit, - int relation, - String userName, - List teamNames) { - int filterRelation = -1; - if (userName != null && filter.getFilterType() == FilterType.MENTIONS) { - filterRelation = MENTIONED_IN.ordinal(); - } - return listThreadsByEntityLink( - entityLink.getFullyQualifiedFieldValue(), - entityLink.getFullyQualifiedFieldType(), - limit, - relation, - userName, - teamNames, - filterRelation, - filter.getCondition(), - filter.getQueryParams()); - } - - @SqlQuery( - "SELECT json FROM " - + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " - + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " - + " ((fromType='user' AND fromFQNHash= :userName) OR" - + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByEntityLink( - @Define("tableName") String tableName, - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType", - original = "toType", - parts = {":toType", ".%"}) - String toType, - @Bind("limit") int limit, - @Bind("relation") int relation, - @BindFQN("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("filterRelation") int filterRelation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM thread_entity " - + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " - + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " - + " ((fromType='user' AND fromFQNHash= :userName) OR" - + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByEntityLink( - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType", - original = "toType", - parts = {":toType", ".%"}) - String toType, - @Bind("limit") int limit, - @Bind("relation") int relation, - @BindFQN("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("filterRelation") int filterRelation, - @Define("condition") String condition, - @BindMap Map params); - - default int listCountThreadsByEntityLink( - String tableName, - FeedFilter filter, - EntityLink entityLink, - int relation, - String userName, - List teamNames) { - int filterRelation = -1; - if (userName != null && filter.getFilterType() == FilterType.MENTIONS) { - filterRelation = MENTIONED_IN.ordinal(); - } - return listCountThreadsByEntityLink( - tableName, - entityLink.getFullyQualifiedFieldValue(), - entityLink.getFullyQualifiedFieldType(), - relation, - userName, - teamNames, - filterRelation, - filter.getCondition(false), - filter.getQueryParams()); - } - - default int listCountThreadsByEntityLink( - FeedFilter filter, - EntityLink entityLink, - int relation, - String userName, - List teamNames) { - int filterRelation = -1; - if (userName != null && filter.getFilterType() == FilterType.MENTIONS) { - filterRelation = MENTIONED_IN.ordinal(); - } - return listCountThreadsByEntityLink( - entityLink.getFullyQualifiedFieldValue(), - entityLink.getFullyQualifiedFieldType(), - relation, - userName, - teamNames, - filterRelation, - filter.getCondition(false), - filter.getQueryParams()); - } - - @SqlQuery( - "SELECT count(id) FROM " - + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " - + "AND (:userName IS NULL OR id in (SELECT toFQNHash FROM field_relationship WHERE " - + " ((fromType='user' AND fromFQNHash= :userName) OR" - + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )") - int listCountThreadsByEntityLink( - @Define("tableName") String tableName, - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType", - original = "toType", - parts = {":toType", ".%"}) - String toType, - @Bind("relation") int relation, - @Bind("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("filterRelation") int filterRelation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT count(id) FROM thread_entity " - + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " - + "AND (:userName IS NULL OR id in (SELECT toFQNHash FROM field_relationship WHERE " - + " ((fromType='user' AND fromFQNHash= :userName) OR" - + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )") - int listCountThreadsByEntityLink( - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType", - original = "toType", - parts = {":toType", ".%"}) - String toType, - @Bind("relation") int relation, - @Bind("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("filterRelation") int filterRelation, - @Define("condition") String condition, - @BindMap Map params); - - @ConnectionAwareSqlUpdate( - value = "UPDATE SET json = :json where id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE SET json = (:json :: jsonb) where id = :id", - connectionType = POSTGRES) - void update( - @Define("tableName") String tableName, @BindUUID("id") UUID id, @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = "UPDATE thread_entity SET json = :json where id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE thread_entity SET json = (:json :: jsonb) where id = :id", - connectionType = POSTGRES) - void update(@BindUUID("id") UUID id, @Bind("json") String json); - - @SqlQuery( - "SELECT entityLink, type, taskStatus, COUNT(id) as count FROM ( " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE hash_id IN ( " - + " SELECT fromFQNHash FROM field_relationship " - + " WHERE " - + " (:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash = :fqnPrefixHash) " - + " AND fromType = 'THREAD' " - + " AND (:toType IS NULL OR toType LIKE :concatToType OR toType = :toType) " - + " AND relation = 3 " - + " ) " - + " UNION " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE te.entityId = :entityId " - + ") AS combined WHERE combined.type IS NOT NULL " - + "GROUP BY type, taskStatus, entityLink") - @RegisterRowMapper(ThreadCountFieldMapper.class) - List> listCountByEntityLink( - @Define("tableName") String tableName, - @BindUUID("entityId") UUID entityId, - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType", - original = "toType", - parts = {":toType", ".%"}) - String toType); - - @SqlQuery( - "SELECT entityLink, type, taskStatus, COUNT(id) as count FROM ( " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE hash_id IN ( " - + " SELECT fromFQNHash FROM field_relationship " - + " WHERE " - + " (:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash = :fqnPrefixHash) " - + " AND fromType = 'THREAD' " - + " AND (:toType IS NULL OR toType LIKE :concatToType OR toType = :toType) " - + " AND relation = 3 " - + " ) " - + " UNION " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE te.entityId = :entityId " - + ") AS combined WHERE combined.type IS NOT NULL " - + "GROUP BY type, taskStatus, entityLink") - @RegisterRowMapper(ThreadCountFieldMapper.class) - List> listCountByEntityLink( - @BindUUID("entityId") UUID entityId, - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType", - original = "toType", - parts = {":toType", ".%"}) - String toType); - - @ConnectionAwareSqlQuery( - value = - "SELECT COUNT(te.id) AS count " - + "FROM te " - + "WHERE te.type = 'Announcement' " - + " AND te.entityLink = :entityLink " - + " AND CAST(JSON_EXTRACT(te.json, '$.announcement.startTime') AS UNSIGNED) <= UNIX_TIMESTAMP()*1000 " - + " AND CAST(JSON_EXTRACT(te.json, '$.announcement.endTime') AS UNSIGNED) >= UNIX_TIMESTAMP()*1000", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT COUNT(te.id) AS count " - + "FROM te " - + "WHERE te.type = 'Announcement' " - + " AND te.entityLink = :entityLink " - + " AND (te.json->'announcement'->>'startTime')::numeric <= EXTRACT(EPOCH FROM NOW()) * 1000 " - + " AND (te.json->'announcement'->>'endTime')::numeric >= EXTRACT(EPOCH FROM NOW()) * 1000", - connectionType = POSTGRES) - int countActiveAnnouncement( - @Define("tableName") String tableName, @Bind("entityLink") String entityLink); - - @ConnectionAwareSqlQuery( - value = - "SELECT COUNT(te.id) AS count " - + "FROM thread_entity te " - + "WHERE te.type = 'Announcement' " - + " AND te.entityLink = :entityLink " - + " AND CAST(JSON_EXTRACT(te.json, '$.announcement.startTime') AS UNSIGNED) <= UNIX_TIMESTAMP()*1000 " - + " AND CAST(JSON_EXTRACT(te.json, '$.announcement.endTime') AS UNSIGNED) >= UNIX_TIMESTAMP()*1000", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT COUNT(te.id) AS count " - + "FROM thread_entity te " - + "WHERE te.type = 'Announcement' " - + " AND te.entityLink = :entityLink " - + " AND (te.json->'announcement'->>'startTime')::numeric <= EXTRACT(EPOCH FROM NOW()) * 1000 " - + " AND (te.json->'announcement'->>'endTime')::numeric >= EXTRACT(EPOCH FROM NOW()) * 1000", - connectionType = POSTGRES) - int countActiveAnnouncement(@Bind("entityLink") String entityLink); - - @ConnectionAwareSqlQuery( - value = - "SELECT combined.type, combined.taskStatus, COUNT(combined.id) AS count " - + "FROM ( " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " JOIN entity_relationship er ON te.entityId = er.toId " - + " WHERE " - + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 8 AND te.type <> 'Task') " - + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 8 AND te.type <> 'Task') " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " JOIN entity_relationship er ON te.id = er.toId " - + " WHERE " - + " er.fromEntity = 'user' AND er.fromId = :userId AND er.toEntity = 'THREAD' AND er.relation IN (1, 2) " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " JOIN entity_relationship er ON te.id = er.toId " - + " WHERE " - + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 11) " - + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 11) " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE te.createdBy = :username " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE MATCH(te.taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) " - + ") AS combined WHERE combined.type is not NULL " - + "GROUP BY combined.type, combined.taskStatus;", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT combined.type, combined.taskStatus, COUNT(combined.id) AS count " - + "FROM ( " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " JOIN entity_relationship er ON te.entityId = er.toId " - + " WHERE " - + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 8 AND te.type <> 'Task') " - + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 8 AND te.type <> 'Task') " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " JOIN entity_relationship er ON te.id = er.toId " - + " WHERE " - + " er.fromEntity = 'user' AND er.fromId = :userId AND er.toEntity = 'THREAD' AND er.relation IN (1, 2) " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " JOIN entity_relationship er ON te.id = er.toId " - + " WHERE " - + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 11) " - + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 11) " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE te.createdBy = :username " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) " - + ") AS combined WHERE combined.type is not NULL " - + "GROUP BY combined.type, combined.taskStatus;", - connectionType = POSTGRES) - @RegisterRowMapper(OwnerCountFieldMapper.class) - List> listCountByOwner( - @Define("tableName") String tableName, - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("username") String username, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres); - - @ConnectionAwareSqlQuery( - value = - "SELECT combined.type, combined.taskStatus, COUNT(combined.id) AS count " - + "FROM ( " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " JOIN entity_relationship er ON te.entityId = er.toId " - + " WHERE " - + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 8 AND te.type <> 'Task') " - + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 8 AND te.type <> 'Task') " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " JOIN entity_relationship er ON te.id = er.toId " - + " WHERE " - + " er.fromEntity = 'user' AND er.fromId = :userId AND er.toEntity = 'THREAD' AND er.relation IN (1, 2) " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " JOIN entity_relationship er ON te.id = er.toId " - + " WHERE " - + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 11) " - + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 11) " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE te.createdBy = :username " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE MATCH(te.taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) " - + ") AS combined WHERE combined.type is not NULL " - + "GROUP BY combined.type, combined.taskStatus;", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT combined.type, combined.taskStatus, COUNT(combined.id) AS count " - + "FROM ( " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " JOIN entity_relationship er ON te.entityId = er.toId " - + " WHERE " - + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 8 AND te.type <> 'Task') " - + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 8 AND te.type <> 'Task') " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " JOIN entity_relationship er ON te.id = er.toId " - + " WHERE " - + " er.fromEntity = 'user' AND er.fromId = :userId AND er.toEntity = 'THREAD' AND er.relation IN (1, 2) " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " JOIN entity_relationship er ON te.id = er.toId " - + " WHERE " - + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 11) " - + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 11) " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE te.createdBy = :username " - + " UNION " - + " SELECT te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) " - + ") AS combined WHERE combined.type is not NULL " - + "GROUP BY combined.type, combined.taskStatus;", - connectionType = POSTGRES) - @RegisterRowMapper(OwnerCountFieldMapper.class) - List> listCountByOwner( - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("username") String username, - @Bind("userTeamJsonMysql") String userTeamJsonMysql, - @Bind("userTeamJsonPostgres") String userTeamJsonPostgres); - - @SqlQuery( - "SELECT json FROM AND " - + "entityId in (" - + "SELECT toId FROM entity_relationship WHERE " - + "((fromEntity='user' AND fromId= :userId) OR " - + "(fromEntity='team' AND fromId IN ())) AND relation= :relation) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByFollows( - @Define("tableName") String tableName, - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("limit") int limit, - @Bind("relation") int relation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM thread_entity AND " - + "entityId in (" - + "SELECT toId FROM entity_relationship WHERE " - + "((fromEntity='user' AND fromId= :userId) OR " - + "(fromEntity='team' AND fromId IN ())) AND relation= :relation) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByFollows( - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("limit") int limit, - @Bind("relation") int relation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT count(id) FROM AND " - + "entityId in (" - + "SELECT toId FROM entity_relationship WHERE " - + "((fromEntity='user' AND fromId= :userId) OR " - + "(fromEntity='team' AND fromId IN ())) AND relation= :relation)") - int listCountThreadsByFollows( - @Define("tableName") String tableName, - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("relation") int relation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT count(id) FROM thread_entity AND " - + "entityId in (" - + "SELECT toId FROM entity_relationship WHERE " - + "((fromEntity='user' AND fromId= :userId) OR " - + "(fromEntity='team' AND fromId IN ())) AND relation= :relation)") - int listCountThreadsByFollows( - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("relation") int relation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM ( " - + " SELECT json, createdAt FROM te " - + " AND entityId IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.relation = 8 " - + " AND ( " - + " (er.fromEntity = 'user' AND er.fromId = :userId) " - + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " - + " ) " - + " ) " - + " UNION " - + " SELECT json, createdAt FROM te " - + " AND id IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.toEntity = 'THREAD' " - + " AND er.relation IN (1, 2) " - + " AND er.fromEntity = 'user' " - + " AND er.fromId = :userId " - + " ) " - + " UNION " - + " SELECT json, createdAt FROM te " - + " AND id IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.relation = 11 " - + " AND ( " - + " (er.fromEntity = 'user' AND er.fromId = :userId) " - + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " - + " ) " - + " ) " - + ") AS combined " - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByOwnerOrFollows( - @Define("tableName") String tableName, - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM ( " - + " SELECT json, createdAt FROM thread_entity te " - + " AND entityId IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.relation = 8 " - + " AND ( " - + " (er.fromEntity = 'user' AND er.fromId = :userId) " - + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " - + " ) " - + " ) " - + " UNION " - + " SELECT json, createdAt FROM thread_entity te " - + " AND id IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.toEntity = 'THREAD' " - + " AND er.relation IN (1, 2) " - + " AND er.fromEntity = 'user' " - + " AND er.fromId = :userId " - + " ) " - + " UNION " - + " SELECT json, createdAt FROM thread_entity te " - + " AND id IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.relation = 11 " - + " AND ( " - + " (er.fromEntity = 'user' AND er.fromId = :userId) " - + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " - + " ) " - + " ) " - + ") AS combined " - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByOwnerOrFollows( - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT COUNT(id) FROM ( " - + " SELECT te.id FROM te " - + " AND entityId IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.relation = 8 " - + " AND ( " - + " (er.fromEntity = 'user' AND er.fromId = :userId) " - + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " - + " ) " - + " ) " - + " UNION " - + " SELECT te.id FROM te " - + " AND id IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.toEntity = 'THREAD' " - + " AND er.relation IN (1, 2) " - + " AND er.fromEntity = 'user' " - + " AND er.fromId = :userId " - + " ) " - + " UNION " - + " SELECT te.id FROM te " - + " AND id IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.relation = 11 " - + " AND ( " - + " (er.fromEntity = 'user' AND er.fromId = :userId) " - + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " - + " ) " - + " ) " - + ") AS combined") - int listCountThreadsByOwnerOrFollows( - @Define("tableName") String tableName, - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT COUNT(id) FROM ( " - + " SELECT te.id FROM thread_entity te " - + " AND entityId IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.relation = 8 " - + " AND ( " - + " (er.fromEntity = 'user' AND er.fromId = :userId) " - + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " - + " ) " - + " ) " - + " UNION " - + " SELECT te.id FROM thread_entity te " - + " AND id IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.toEntity = 'THREAD' " - + " AND er.relation IN (1, 2) " - + " AND er.fromEntity = 'user' " - + " AND er.fromId = :userId " - + " ) " - + " UNION " - + " SELECT te.id FROM thread_entity te " - + " AND id IN ( " - + " SELECT toId FROM entity_relationship er " - + " WHERE er.relation = 11 " - + " AND ( " - + " (er.fromEntity = 'user' AND er.fromId = :userId) " - + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " - + " ) " - + " ) " - + ") AS combined") - int listCountThreadsByOwnerOrFollows( - @BindUUID("userId") UUID userId, - @BindList("teamIds") List teamIds, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM AND " - + "hash_id in (" - + "SELECT toFQNHash FROM field_relationship WHERE " - + "((fromType='user' AND fromFQNHash= :userName) OR " - + "(fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :relation) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByMentions( - @Define("tableName") String tableName, - @Bind("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("limit") int limit, - @Bind("relation") int relation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM thread_entity AND " - + "hash_id in (" - + "SELECT toFQNHash FROM field_relationship WHERE " - + "((fromType='user' AND fromFQNHash= :userName) OR " - + "(fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :relation) " - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByMentions( - @Bind("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("limit") int limit, - @Bind("relation") int relation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT count(id) FROM AND " - + "hash_id in (" - + "SELECT toFQNHash FROM field_relationship WHERE " - + "((fromType='user' AND fromFQNHash= :userName) OR " - + "(fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :relation) ") - int listCountThreadsByMentions( - @Define("tableName") String tableName, - @Bind("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("relation") int relation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT count(id) FROM thread_entity AND " - + "hash_id in (" - + "SELECT toFQNHash FROM field_relationship WHERE " - + "((fromType='user' AND fromFQNHash= :userName) OR " - + "(fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :relation) ") - int listCountThreadsByMentions( - @Bind("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("relation") int relation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM " - + "AND MD5(id) in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "((:toType1 IS NULL OR toType LIKE :concatToType1 OR toType=:toType1) OR " - + "(:toType2 IS NULL OR toType LIKE :concatToType2 OR toType=:toType2)) AND relation= :relation)" - + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " - + " ((fromType='user' AND fromFQNHash= :userName) OR" - + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByGlossaryAndTerms( - @Define("tableName") String tableName, - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType1", - original = "toType1", - parts = {":toType1", ".%"}) - String toType1, - @BindConcat( - value = "concatToType2", - original = "toType2", - parts = {":toType2", ".%"}) - String toType2, - @Bind("limit") int limit, - @Bind("relation") int relation, - @BindFQN("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("filterRelation") int filterRelation, - @Define("condition") String condition, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM thread_entity " - + "AND MD5(id) in (SELECT fromFQNHash FROM field_relationship WHERE " - + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " - + "((:toType1 IS NULL OR toType LIKE :concatToType1 OR toType=:toType1) OR " - + "(:toType2 IS NULL OR toType LIKE :concatToType2 OR toType=:toType2)) AND relation= :relation)" - + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " - + " ((fromType='user' AND fromFQNHash= :userName) OR" - + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" - + "ORDER BY createdAt DESC " - + "LIMIT :limit") - List listThreadsByGlossaryAndTerms( - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType1", - original = "toType1", - parts = {":toType1", ".%"}) - String toType1, - @BindConcat( - value = "concatToType2", - original = "toType2", - parts = {":toType2", ".%"}) - String toType2, - @Bind("limit") int limit, - @Bind("relation") int relation, - @BindFQN("userName") String userName, - @BindList("teamNames") List teamNames, - @Bind("filterRelation") int filterRelation, - @Define("condition") String condition, - @BindMap Map params); - - default List> listCountThreadsByGlossaryAndTerms( - String tableName, EntityLink entityLink, EntityReference reference) { - EntityLink glossaryTermLink = - new EntityLink(GLOSSARY_TERM, entityLink.getFullyQualifiedFieldValue()); - return listCountThreadsByGlossaryAndTerms( - tableName, - reference.getId(), - reference.getFullyQualifiedName(), - entityLink.getFullyQualifiedFieldType(), - glossaryTermLink.getFullyQualifiedFieldType()); - } - - default List> listCountThreadsByGlossaryAndTerms( - EntityLink entityLink, EntityReference reference) { - EntityLink glossaryTermLink = - new EntityLink(GLOSSARY_TERM, entityLink.getFullyQualifiedFieldValue()); - return listCountThreadsByGlossaryAndTerms( - reference.getId(), - reference.getFullyQualifiedName(), - entityLink.getFullyQualifiedFieldType(), - glossaryTermLink.getFullyQualifiedFieldType()); - } - - default List listThreadsByTaskAssignee(String taskAssigneesId) { - return listThreadsByTaskAssigneesId("%" + taskAssigneesId + "%"); - } - - @SqlQuery("SELECT json FROM WHERE taskAssigneesIds LIKE :taskAssigneesPattern") - List listThreadsByTaskAssigneesId( - @Define("tableName") String tableName, - @Bind("taskAssigneesPattern") String taskAssigneesPattern); - - @SqlQuery("SELECT json FROM thread_entity WHERE taskAssigneesIds LIKE :taskAssigneesPattern") - List listThreadsByTaskAssigneesId( - @Bind("taskAssigneesPattern") String taskAssigneesPattern); - - @SqlQuery( - "SELECT entityLink, type, taskStatus, COUNT(id) as count " - + "FROM ( " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE te.entityId = :entityId " - + " UNION " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE te.hash_id IN ( " - + " SELECT fr.fromFQNHash " - + " FROM field_relationship fr " - + " WHERE (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " - + " AND fr.fromType = 'THREAD' " - + " AND (:toType1 IS NULL OR fr.toType LIKE :concatToType1 OR fr.toType = :toType1) " - + " AND fr.relation = 3 " - + " ) " - + " UNION " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM te " - + " WHERE te.type = 'Task' " - + " AND te.hash_id IN ( " - + " SELECT fr.fromFQNHash " - + " FROM field_relationship fr " - + " JOIN te2 ON te2.hash_id = fr.fromFQNHash WHERE fr.fromFQNHash = te.hash_id AND te2.type = 'Task' " - + " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " - + " AND fr.fromType = 'THREAD' " - + " AND (:toType2 IS NULL OR fr.toType LIKE :concatToType2 OR fr.toType = :toType2) " - + " AND fr.relation = 3 " - + " ) " - + ") AS combined_results WHERE combined_results.type is not NULL " - + "GROUP BY entityLink, type, taskStatus ") - @RegisterRowMapper(ThreadCountFieldMapper.class) - List> listCountThreadsByGlossaryAndTerms( - @Define("tableName") String tableName, - @BindUUID("entityId") UUID entityId, - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType1", - original = "toType1", - parts = {":toType1", ".%"}) - String toType1, - @BindConcat( - value = "concatToType2", - original = "toType2", - parts = {":toType2", ".%"}) - String toType2); - - @SqlQuery( - "SELECT entityLink, type, taskStatus, COUNT(id) as count " - + "FROM ( " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE te.entityId = :entityId " - + " UNION " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE te.hash_id IN ( " - + " SELECT fr.fromFQNHash " - + " FROM field_relationship fr " - + " WHERE (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " - + " AND fr.fromType = 'THREAD' " - + " AND (:toType1 IS NULL OR fr.toType LIKE :concatToType1 OR fr.toType = :toType1) " - + " AND fr.relation = 3 " - + " ) " - + " UNION " - + " SELECT te.entityLink, te.type, te.taskStatus, te.id " - + " FROM thread_entity te " - + " WHERE te.type = 'Task' " - + " AND te.hash_id IN ( " - + " SELECT fr.fromFQNHash " - + " FROM field_relationship fr " - + " JOIN thread_entity te2 ON te2.hash_id = fr.fromFQNHash WHERE fr.fromFQNHash = te.hash_id AND te2.type = 'Task' " - + " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " - + " AND fr.fromType = 'THREAD' " - + " AND (:toType2 IS NULL OR fr.toType LIKE :concatToType2 OR fr.toType = :toType2) " - + " AND fr.relation = 3 " - + " ) " - + ") AS combined_results WHERE combined_results.type is not NULL " - + "GROUP BY entityLink, type, taskStatus ") - @RegisterRowMapper(ThreadCountFieldMapper.class) - List> listCountThreadsByGlossaryAndTerms( - @BindUUID("entityId") UUID entityId, - @BindConcat( - value = "concatFqnPrefixHash", - original = "fqnPrefixHash", - parts = {":fqnPrefixHash", ".%"}, - hash = true) - String fqnPrefixHash, - @BindConcat( - value = "concatToType1", - original = "toType1", - parts = {":toType1", ".%"}) - String toType1, - @BindConcat( - value = "concatToType2", - original = "toType2", - parts = {":toType2", ".%"}) - String toType2); - - @SqlQuery("select id from where entityId = :entityId") - List findByEntityId( - @Define("tableName") String tableName, @Bind("entityId") String entityId); - - @SqlQuery("select id from thread_entity where entityId = :entityId") - List findByEntityId(@Bind("entityId") String entityId); - - // DISTINCT is defence-in-depth: thread_entity.id is a primary key, and entityId is a - // single-valued column per row, so a single matching scan can't physically return the - // same id twice. The DISTINCT survives a future schema where a thread row picks up - // multiple entity references (or a join is added) — keeping the consumer code in - // deleteByAbout from re-issuing redundant relationship / extension / feed deletes for - // the same id under chunking. - @SqlQuery("select DISTINCT id from where entityId IN ()") - List findByEntityIds( - @Define("tableName") String tableName, @BindList("entityIds") List entityIds); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE SET json = JSON_SET(json, '$.about', :newEntityLink)\n" - + "WHERE entityId = :entityId", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE SET json = jsonb_set(json, '{about}', to_jsonb(:newEntityLink::text), false)\n" - + "WHERE entityId = :entityId", - connectionType = POSTGRES) - void updateByEntityId( - @Define("tableName") String tableName, - @Bind("newEntityLink") String newEntityLink, - @Bind("entityId") String entityId); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE thread_entity SET json = JSON_SET(json, '$.about', :newEntityLink)\n" - + "WHERE entityId = :entityId", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE thread_entity SET json = jsonb_set(json, '{about}', to_jsonb(:newEntityLink::text), false)\n" - + "WHERE entityId = :entityId", - connectionType = POSTGRES) - void updateByEntityId( - @Bind("newEntityLink") String newEntityLink, @Bind("entityId") String entityId); - - class OwnerCountFieldMapper implements RowMapper> { - @Override - public List map(ResultSet rs, StatementContext ctx) throws SQLException { - return Arrays.asList( - rs.getString("type"), rs.getString("taskStatus"), rs.getString("count")); - } - } - - class ThreadCountFieldMapper implements RowMapper> { - @Override - public List map(ResultSet rs, StatementContext ctx) throws SQLException { - return Arrays.asList( - rs.getString("entityLink"), - rs.getString("type"), - rs.getString("taskStatus"), - rs.getString("count")); - } - } - } - - interface TaskDAO extends EntityDAO { - class TaskCountSummary { - private final int total; - private final int open; - private final int completed; - private final int inProgress; - private final int approved; - private final int granted; - - public TaskCountSummary( - int total, int open, int completed, int inProgress, int approved, int granted) { - this.total = total; - this.open = open; - this.completed = completed; - this.inProgress = inProgress; - this.approved = approved; - this.granted = granted; - } - - public int getTotal() { - return total; - } - - public int getOpen() { - return open; - } - - public int getCompleted() { - return completed; - } - - public int getInProgress() { - return inProgress; - } - - public int getApproved() { - return approved; - } - - public int getGranted() { - return granted; - } - } - - class TaskCountSummaryMapper implements RowMapper { - @Override - public TaskCountSummary map(ResultSet rs, StatementContext ctx) throws SQLException { - return new TaskCountSummary( - rs.getInt("total"), - rs.getInt("openCount"), - rs.getInt("completedCount"), - rs.getInt("inProgressCount"), - rs.getInt("approvedCount"), - rs.getInt("grantedCount")); - } - } - - @Override - default String getTableName() { - return "task_entity"; - } - - @Override - default Class getEntityClass() { - return Task.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @ConnectionAwareSqlUpdate( - value = "INSERT INTO task_entity (id, json, fqnHash) VALUES (:id, :json, :fqnHash)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO task_entity (id, json, fqnHash) VALUES (:id, :json :: jsonb, :fqnHash)", - connectionType = POSTGRES) - void insertTask( - @Bind("id") String id, @Bind("json") String json, @BindFQN("fqnHash") String fqn); - - @Override - default void insert(org.openmetadata.schema.EntityInterface entity, String fqn) { - Task task = (Task) entity; - insertTask(task.getId().toString(), JsonUtils.pojoToJson(task), task.getFullyQualifiedName()); - } - - @ConnectionAwareSqlUpdate( - value = "UPDATE task_entity SET json = :json WHERE id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE task_entity SET json = (:json :: jsonb) WHERE id = :id", - connectionType = POSTGRES) - void updateTask(@Bind("id") String id, @Bind("json") String json); - - @Override - default void update(UUID id, String fqn, String json) { - updateTask(id.toString(), json); - } - - @SqlUpdate("UPDATE new_task_sequence SET id = LAST_INSERT_ID(id + 1)") - int incrementSequenceMysql(); - - @SqlQuery("SELECT LAST_INSERT_ID()") - long getLastInsertIdMysql(); - - @SqlQuery("UPDATE new_task_sequence SET id = id + 1 RETURNING id") - long getNextTaskIdPostgres(); - - @SqlUpdate("DELETE FROM entity_relationship WHERE fromEntity = 'task' OR toEntity = 'task'") - void deleteTaskRelationships(); - - @SqlUpdate("DELETE FROM task_entity") - void deleteAll(); - - @SqlUpdate("UPDATE new_task_sequence SET id = 0") - void resetSequence(); - - @SqlUpdate( - "DELETE FROM entity_relationship WHERE fromEntity = 'domain' AND toEntity = 'task' " - + "AND relation = 10 AND toId IN ()") - void bulkRemoveDomainRelationships(@BindList("taskIds") List taskIds); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM task_entity " - + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.payload.testCaseResolutionStatusId')) = :stateId " - + "AND (JSON_EXTRACT(json, '$.deleted') = false OR JSON_EXTRACT(json, '$.deleted') IS NULL)", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM task_entity " - + "WHERE json->'payload'->>'testCaseResolutionStatusId' = :stateId " - + "AND ((json->>'deleted')::boolean = false OR json->>'deleted' IS NULL)", - connectionType = POSTGRES) - String fetchTaskByTestCaseResolutionStatusId(@Bind("stateId") String stateId); - - @SqlQuery( - "SELECT json FROM task_entity " - + "WHERE aboutFqnHash = :aboutFqnHash AND type = :type " - + "AND status IN () " - + "AND (deleted = false OR deleted IS NULL) " - + "ORDER BY createdAt DESC LIMIT 1") - String findByAboutAndTypeAndStatuses( - @BindFQN("aboutFqnHash") String aboutFqn, - @Bind("type") String type, - @BindList("statuses") List statuses); - - @SqlQuery( - "SELECT json FROM task_entity " - + "WHERE aboutFqnHash = :aboutFqnHash AND type = :type AND status = :status " - + "AND (deleted = false OR deleted IS NULL) " - + "LIMIT 1") - String findByAboutAndTypeAndStatus( - @BindFQN("aboutFqnHash") String aboutFqn, - @Bind("type") String type, - @Bind("status") String status); - - @SqlQuery( - "SELECT json FROM task_entity " - + "WHERE aboutFqnHash = :aboutFqnHash AND category = :category AND status = :status " - + "AND (deleted = false OR deleted IS NULL) " - + "LIMIT 1") - String findByAboutAndCategoryAndStatus( - @BindFQN("aboutFqnHash") String aboutFqn, - @Bind("category") String category, - @Bind("status") String status); - - @SqlUpdate( - "DELETE FROM task_entity " + "WHERE createdById = :createdById AND category = :category") - void deleteByCreatorAndCategory( - @Bind("createdById") String createdById, @Bind("category") String category); - - @ConnectionAwareSqlQuery( - value = - "SELECT id, json_unquote(json_extract(json, '$.fullyQualifiedName')) AS fqn " - + "FROM task_entity WHERE createdById = :createdById AND category = :category", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT id, json->>'fullyQualifiedName' AS fqn " - + "FROM task_entity WHERE createdById = :createdById AND category = :category", - connectionType = POSTGRES) - @RegisterRowMapper(EntityDAO.EntityIdFqnPairMapper.class) - List listIdAndFqnByCreatorAndCategory( - @Bind("createdById") String createdById, @Bind("category") String category); - - @RegisterRowMapper(TaskCountSummaryMapper.class) - @SqlQuery( - // 'Approved' double-counts in `completedCount` AND `approvedCount` because the - // same status means different things across task types: terminal for - // Glossary/DescriptionUpdate (legacy dashboards expect it under "completed") and - // non-terminal for Data Access Requests (the dedicated DAR list uses - // `approvedCount` / `grantedCount` and the `active` status group instead). - // See ListFilter.getTaskStatusCondition for the matching status-group semantics. - "SELECT " - + "COUNT(id) AS total, " - + "COALESCE(SUM(CASE WHEN status IN ('Open', 'InProgress', 'Pending') THEN 1 ELSE 0 END), 0) AS openCount, " - + "COALESCE(SUM(CASE WHEN status IN ('Approved', 'Rejected', 'Completed', 'Cancelled', 'Failed', 'Revoked') THEN 1 ELSE 0 END), 0) AS completedCount, " - + "COALESCE(SUM(CASE WHEN status = 'InProgress' THEN 1 ELSE 0 END), 0) AS inProgressCount, " - + "COALESCE(SUM(CASE WHEN status = 'Approved' THEN 1 ELSE 0 END), 0) AS approvedCount, " - + "COALESCE(SUM(CASE WHEN status = 'Granted' THEN 1 ELSE 0 END), 0) AS grantedCount " - + "FROM task_entity ") - TaskCountSummary getTaskCountSummary( - @Define("condition") String condition, @BindMap Map params); - - @SqlQuery( - "SELECT json FROM task_entity " - + "ORDER BY createdAt , id " - + "LIMIT :limit OFFSET :offset") - List listTasksByCreatedAt( - @Define("cond") String cond, - @BindMap Map params, - @Define("sortOrder") String sortOrder, - @Bind("limit") int limit, - @Bind("offset") int offset); - - @SqlQuery("SELECT count(*) FROM task_entity ") - int listTasksByCreatedAtCount(@Define("cond") String cond, @BindMap Map params); - } - - interface AnnouncementDAO extends EntityDAO { - @Override - default String getTableName() { - return "announcement_entity"; - } - - @Override - default Class getEntityClass() { - return Announcement.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @ConnectionAwareSqlUpdate( - value = "INSERT INTO announcement_entity (id, json, fqnHash) VALUES (:id, :json, :fqnHash)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO announcement_entity (id, json, fqnHash) VALUES (:id, :json :: jsonb, :fqnHash)", - connectionType = POSTGRES) - void insertAnnouncement( - @Bind("id") String id, @Bind("json") String json, @BindFQN("fqnHash") String fqn); - - @Override - default void insert(org.openmetadata.schema.EntityInterface entity, String fqn) { - Announcement announcement = (Announcement) entity; - insertAnnouncement( - announcement.getId().toString(), - JsonUtils.pojoToJson(announcement), - announcement.getFullyQualifiedName()); - } - - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM announcement_entity " - + "WHERE " - + "AND (:entityLink IS NULL OR entityLink = :entityLink) " - + "AND (:status IS NULL OR status = :status) " - + "AND ((:active IS NULL) " - + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " - + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs)))", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM announcement_entity " - + "WHERE " - + "AND (:entityLink IS NULL OR entityLink = :entityLink) " - + "AND (:status IS NULL OR status = :status) " - + "AND ((:active IS NULL) " - + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " - + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs)))", - connectionType = POSTGRES) - int listAnnouncementCount( - @Define("condition") String condition, - @Bind("entityLink") String entityLink, - @Bind("status") String status, - @Bind("active") Boolean active, - @Bind("currentTs") long currentTs); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM announcement_entity " - + "WHERE " - + "AND (:entityLink IS NULL OR entityLink = :entityLink) " - + "AND (:status IS NULL OR status = :status) " - + "AND ((:active IS NULL) " - + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " - + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " - + "ORDER BY name, id LIMIT :limit OFFSET :offset", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM announcement_entity " - + "WHERE " - + "AND (:entityLink IS NULL OR entityLink = :entityLink) " - + "AND (:status IS NULL OR status = :status) " - + "AND ((:active IS NULL) " - + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " - + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " - + "ORDER BY name, id LIMIT :limit OFFSET :offset", - connectionType = POSTGRES) - List listAnnouncementsWithOffset( - @Define("condition") String condition, - @Bind("entityLink") String entityLink, - @Bind("status") String status, - @Bind("active") Boolean active, - @Bind("currentTs") long currentTs, - @Bind("limit") int limit, - @Bind("offset") int offset); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT announcement_entity.name, announcement_entity.id, announcement_entity.json " - + "FROM announcement_entity " - + "WHERE " - + "AND (:entityLink IS NULL OR entityLink = :entityLink) " - + "AND (:status IS NULL OR status = :status) " - + "AND ((:active IS NULL) " - + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " - + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " - + "AND (announcement_entity.name < :beforeName " - + "OR (announcement_entity.name = :beforeName AND announcement_entity.id < :beforeId)) " - + "ORDER BY announcement_entity.name DESC, announcement_entity.id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name, id", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT announcement_entity.name, announcement_entity.id, announcement_entity.json " - + "FROM announcement_entity " - + "WHERE " - + "AND (:entityLink IS NULL OR entityLink = :entityLink) " - + "AND (:status IS NULL OR status = :status) " - + "AND ((:active IS NULL) " - + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " - + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " - + "AND (announcement_entity.name < :beforeName " - + "OR (announcement_entity.name = :beforeName AND announcement_entity.id < :beforeId)) " - + "ORDER BY announcement_entity.name DESC, announcement_entity.id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name, id", - connectionType = POSTGRES) - List listAnnouncementsBefore( - @Define("condition") String condition, - @Bind("entityLink") String entityLink, - @Bind("status") String status, - @Bind("active") Boolean active, - @Bind("currentTs") long currentTs, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId); - - @ConnectionAwareSqlQuery( - value = - "SELECT announcement_entity.json FROM announcement_entity " - + "WHERE " - + "AND (:entityLink IS NULL OR entityLink = :entityLink) " - + "AND (:status IS NULL OR status = :status) " - + "AND ((:active IS NULL) " - + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " - + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " - + "AND (announcement_entity.name > :afterName " - + "OR (announcement_entity.name = :afterName AND announcement_entity.id > :afterId)) " - + "ORDER BY announcement_entity.name, announcement_entity.id " - + "LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT announcement_entity.json FROM announcement_entity " - + "WHERE " - + "AND (:entityLink IS NULL OR entityLink = :entityLink) " - + "AND (:status IS NULL OR status = :status) " - + "AND ((:active IS NULL) " - + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " - + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " - + "AND (announcement_entity.name > :afterName " - + "OR (announcement_entity.name = :afterName AND announcement_entity.id > :afterId)) " - + "ORDER BY announcement_entity.name, announcement_entity.id " - + "LIMIT :limit", - connectionType = POSTGRES) - List listAnnouncementsAfter( - @Define("condition") String condition, - @Bind("entityLink") String entityLink, - @Bind("status") String status, - @Bind("active") Boolean active, - @Bind("currentTs") long currentTs, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId); - - private String getAnnouncementBaseCondition(ListFilter filter) { - String includeCondition = filter.getIncludeCondition(getTableName()); - return includeCondition.isEmpty() ? "TRUE" : includeCondition; - } - - private Boolean getActiveFlag(ListFilter filter) { - String active = filter.getQueryParam("active"); - return active == null ? null : Boolean.parseBoolean(active); - } - - private String getAnnouncementStatus(ListFilter filter) { - return filter.getQueryParam("status"); - } - - private String getAnnouncementEntityLink(ListFilter filter) { - return filter.getQueryParam("entityLink"); - } - - @Override - default int listCount(ListFilter filter) { - if (filter.getQueryParam("active") == null) { - return EntityDAO.super.listCount(filter); - } - - return listAnnouncementCount( - getAnnouncementBaseCondition(filter), - getAnnouncementEntityLink(filter), - getAnnouncementStatus(filter), - getActiveFlag(filter), - System.currentTimeMillis()); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - if (filter.getQueryParam("active") == null) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - - return listAnnouncementsBefore( - getAnnouncementBaseCondition(filter), - getAnnouncementEntityLink(filter), - getAnnouncementStatus(filter), - getActiveFlag(filter), - System.currentTimeMillis(), - limit, - beforeName, - beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - if (filter.getQueryParam("active") == null) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - - return listAnnouncementsAfter( - getAnnouncementBaseCondition(filter), - getAnnouncementEntityLink(filter), - getAnnouncementStatus(filter), - getActiveFlag(filter), - System.currentTimeMillis(), - limit, - afterName, - afterId); - } - - @Override - default List listAfter(ListFilter filter, int limit, int offset) { - if (filter.getQueryParam("active") == null) { - return EntityDAO.super.listAfter(filter, limit, offset); - } - - return listAnnouncementsWithOffset( - getAnnouncementBaseCondition(filter), - getAnnouncementEntityLink(filter), - getAnnouncementStatus(filter), - getActiveFlag(filter), - System.currentTimeMillis(), - limit, - offset); - } - } - - interface TaskFormSchemaDAO extends EntityDAO { - @Override - default String getTableName() { - return "task_form_schema_entity"; - } - - @Override - default Class getEntityClass() { - return TaskFormSchema.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO task_form_schema_entity (id, json, fqnHash) VALUES (:id, :json, :fqnHash)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO task_form_schema_entity (id, json, fqnHash) VALUES (:id, :json :: jsonb, :fqnHash)", - connectionType = POSTGRES) - void insertTaskFormSchema( - @Bind("id") String id, @Bind("json") String json, @BindFQN("fqnHash") String fqn); - - @Override - default void insert(org.openmetadata.schema.EntityInterface entity, String fqn) { - TaskFormSchema schema = (TaskFormSchema) entity; - insertTaskFormSchema( - schema.getId().toString(), JsonUtils.pojoToJson(schema), schema.getFullyQualifiedName()); - } - } - - interface FieldRelationshipDAO { - @ConnectionAwareSqlUpdate( - value = - "INSERT IGNORE INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, json) " - + "VALUES (:fromFQNHash, :toFQNHash, :fromFQN, :toFQN, :fromType, :toType, :relation, :json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, json) " - + "VALUES (:fromFQNHash, :toFQNHash, :fromFQN, :toFQN, :fromType, :toType, :relation, (:json :: jsonb)) " - + "ON CONFLICT (fromFQNHash, toFQNHash, relation) DO NOTHING", - connectionType = POSTGRES) - void insert( - @BindFQN("fromFQNHash") String fromFQNHash, - @BindFQN("toFQNHash") String toFQNHash, - @Bind("fromFQN") String fromFQN, - @Bind("toFQN") String toFQN, - @Bind("fromType") String fromType, - @Bind("toType") String toType, - @Bind("relation") int relation, - @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, jsonSchema, json) " - + "VALUES (:fromFQNHash, :toFQNHash, :fromFQN, :toFQN, :fromType, :toType, :relation, :jsonSchema, :json) " - + "ON DUPLICATE KEY UPDATE json = :json", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, jsonSchema, json) " - + "VALUES (:fromFQNHash, :toFQNHash, :fromFQN, :toFQN, :fromType, :toType, :relation, :jsonSchema, (:json :: jsonb)) " - + "ON CONFLICT (fromFQNHash, toFQNHash, relation) DO UPDATE SET json = EXCLUDED.json", - connectionType = POSTGRES) - void upsert( - @BindFQN("fromFQNHash") String fromFQNHash, - @BindFQN("toFQNHash") String toFQNHash, - @Bind("fromFQN") String fromFQN, - @Bind("toFQN") String toFQN, - @Bind("fromType") String fromType, - @Bind("toType") String toType, - @Bind("relation") int relation, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json); - - @SqlQuery( - "SELECT json FROM field_relationship WHERE " - + "fromFQNHash = :fromFQNHash AND toFQNHash = :toFQNHash AND fromType = :fromType " - + "AND toType = :toType AND relation = :relation") - String find( - @BindFQN("fromFQNHash") String fromFQNHash, - @BindFQN("toFQNHash") String toFQNHash, - @Bind("fromType") String fromType, - @Bind("toType") String toType, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT fromFQN, fromType, json FROM field_relationship WHERE " - + "toFQNHash = :toFQNHash AND toType = :toType AND relation = :relation") - @RegisterRowMapper(FromFieldMapper.class) - List> findFrom( - @BindFQN("toFQNHash") String toFQNHash, - @Bind("toType") String toType, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT fromFQN, toFQN, json FROM field_relationship WHERE " - + "fromFQNHash LIKE :concatFqnPrefixHash AND fromType = :fromType AND toType = :toType " - + "AND relation = :relation") - @RegisterRowMapper(ToFieldMapper.class) - List> listToByPrefix( - @BindConcat( - value = "concatFqnPrefixHash", - parts = {":fqnPrefixHash", "%"}, - hash = true) - String fqnPrefixHash, - @Bind("fromType") String fromType, - @Bind("toType") String toType, - @Bind("relation") int relation); - - @Deprecated(since = "Release 1.1") - @SqlQuery( - "SELECT DISTINCT fromFQN, toFQN FROM field_relationship WHERE fromFQNHash = '' or fromFQNHash is null or toFQNHash = '' or toFQNHash is null LIMIT :limit") - @RegisterRowMapper(FieldRelationShipMapper.class) - List> migrationListDistinctWithOffset(@Bind("limit") int limit); - - @SqlQuery( - "SELECT fromFQN, toFQN, json FROM field_relationship WHERE " - + "fromFQNHash = :fqnHash AND fromType = :type AND toType = :otherType AND relation = :relation " - + "UNION " - + "SELECT toFQN, fromFQN, json FROM field_relationship WHERE " - + "toFQNHash = :fqnHash AND toType = :type AND fromType = :otherType AND relation = :relation") - @RegisterRowMapper(ToFieldMapper.class) - List> listBidirectional( - @BindFQN("fqnHash") String fqnHash, - @Bind("type") String type, - @Bind("otherType") String otherType, - @Bind("relation") int relation); - - @SqlQuery( - "SELECT fromFQN, toFQN, json FROM field_relationship WHERE " - + "fromFQNHash LIKE :concatFqnPrefixHash AND fromType = :type AND toType = :otherType AND relation = :relation " - + "UNION " - + "SELECT toFQN, fromFQN, json FROM field_relationship WHERE " - + "toFQNHash LIKE :concatFqnPrefixHash AND toType = :type AND fromType = :otherType AND relation = :relation") - @RegisterRowMapper(ToFieldMapper.class) - List> listBidirectionalByPrefix( - @BindConcat( - value = "concatFqnPrefixHash", - parts = {":fqnPrefixHash", "%"}, - hash = true) - String fqnPrefixHash, - @Bind("type") String type, - @Bind("otherType") String otherType, - @Bind("relation") int relation); - - default void deleteAllByPrefix(String fqn) { - String prefix = String.format("%s%s%%", FullyQualifiedName.buildHash(fqn), Entity.SEPARATOR); - String condition = "WHERE (toFQNHash LIKE :prefix OR fromFQNHash LIKE :prefix)"; - Map bindMap = new HashMap<>(); - bindMap.put("prefix", prefix); - deleteAllByPrefixInternal(condition, bindMap); - } - - default void deleteAllByPrefixes(List threadIds) { - for (String threadId : threadIds) { - deleteAllByPrefix(threadId); - } - } - - @SqlUpdate("DELETE from field_relationship ") - void deleteAllByPrefixInternal( - @Define("cond") String cond, @BindMap Map bindings); - - @SqlUpdate( - "DELETE from field_relationship WHERE fromFQNHash = :fromFQNHash AND toFQNHash = :toFQNHash AND fromType = :fromType " - + "AND toType = :toType AND relation = :relation") - void delete( - @BindFQN("fromFQNHash") String fromFQNHash, - @BindFQN("toFQNHash") String toFQNHash, - @Bind("fromType") String fromType, - @Bind("toType") String toType, - @Bind("relation") int relation); - - default void renameByToFQN(String oldToFQN, String newToFQN) { - renameByToFQNInternal( - oldToFQN, - FullyQualifiedName.buildHash(oldToFQN), - newToFQN, - FullyQualifiedName.buildHash(newToFQN)); // First rename targetFQN from oldFQN to newFQN - renameByToFQNPrefix(oldToFQN, newToFQN); - // Rename all the targetFQN prefixes starting with the oldFQN to newFQN - } - - @SqlUpdate( - "Update field_relationship set toFQN = :newToFQN , toFQNHash = :newToFQNHash " - + "where fromtype = 'THREAD' AND relation='3' AND toFQN = :oldToFQN and toFQNHash =:oldToFQNHash ;") - void renameByToFQNInternal( - @Bind("oldToFQN") String oldToFQN, - @Bind("oldToFQNHash") String oldToFQNHash, - @Bind("newToFQN") String newToFQN, - @Bind("newToFQNHash") String newToFQNHash); - - default void renameByToFQNPrefix(String oldToFQNPrefix, String newToFQNPrefix) { - String update = - String.format( - "UPDATE field_relationship SET toFQN = REPLACE(toFQN, '%s.', '%s.') , toFQNHash = REPLACE(toFQNHash, '%s.', '%s.') where fromtype = 'THREAD' AND relation='3' AND toFQN like '%s.%%' and toFQNHash like '%s.%%' ", - escapeApostrophe(oldToFQNPrefix), - escapeApostrophe(newToFQNPrefix), - FullyQualifiedName.buildHash(oldToFQNPrefix), - FullyQualifiedName.buildHash(newToFQNPrefix), - escapeApostrophe(oldToFQNPrefix), - FullyQualifiedName.buildHash(oldToFQNPrefix)); - renameByToFQNPrefixInternal(update); - } - - @SqlUpdate("") - void renameByToFQNPrefixInternal(@Define("update") String update); - - class FromFieldMapper implements RowMapper> { - @Override - public Triple map(ResultSet rs, StatementContext ctx) - throws SQLException { - return Triple.of(rs.getString("fromFQN"), rs.getString("fromType"), rs.getString("json")); - } - } - - class ToFieldMapper implements RowMapper> { - @Override - public Triple map(ResultSet rs, StatementContext ctx) - throws SQLException { - return Triple.of(rs.getString("fromFQN"), rs.getString("toFQN"), rs.getString("json")); - } - } - - class FieldRelationShipMapper implements RowMapper> { - @Override - public Pair map(ResultSet rs, StatementContext ctx) throws SQLException { - return Pair.of(rs.getString("fromFQN"), rs.getString("toFQN")); - } - } - - @Getter - @Setter - class FieldRelationship { - private String fromFQNHash; - private String toFQNHash; - private String fromFQN; - private String toFQN; - private String fromType; - private String toType; - private int relation; - private String jsonSchema; - private String json; - } - } - - interface BotDAO extends EntityDAO { - @Override - default String getTableName() { - return "bot_entity"; - } - - @Override - default Class getEntityClass() { - return Bot.class; - } - } - - interface DomainDAO extends EntityDAO { - @Override - default String getTableName() { - return "domain_entity"; - } - - @Override - default Class getEntityClass() { - return Domain.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default boolean supportsSoftDelete() { - return false; - } - - @Override - default int listCount(ListFilter filter) { - String condition = filter.getCondition(); - String directChildrenOf = filter.getQueryParam("directChildrenOf"); - String hierarchyFilter = filter.getQueryParam("hierarchyFilter"); - - if (!nullOrEmpty(directChildrenOf)) { - String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); - filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); - filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); - - condition += - " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; - } else if (Boolean.TRUE.toString().equals(hierarchyFilter)) { - // For hierarchy API, when directChildrenOf is null, show only root domains - condition += - " AND NOT EXISTS (SELECT 1 FROM entity_relationship er WHERE er.toId = domain_entity.id AND er.fromEntity = 'domain' AND er.toEntity = 'domain' AND er.relation = " - + Relationship.CONTAINS.ordinal() - + ")"; - } - - return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), condition); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String condition = filter.getCondition(); - String directChildrenOf = filter.getQueryParam("directChildrenOf"); - String hierarchyFilter = filter.getQueryParam("hierarchyFilter"); - - if (!nullOrEmpty(directChildrenOf)) { - String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); - filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); - filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); - - condition += - " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; - } else if (Boolean.TRUE.toString().equals(hierarchyFilter)) { - // For hierarchy API, when directChildrenOf is null, show only root domains - condition += - " AND NOT EXISTS (SELECT 1 FROM entity_relationship er WHERE er.toId = domain_entity.id AND er.fromEntity = 'domain' AND er.toEntity = 'domain' AND er.relation = " - + Relationship.CONTAINS.ordinal() - + ")"; - } - - return listBefore( - getTableName(), filter.getQueryParams(), condition, limit, beforeName, beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String condition = filter.getCondition(); - String directChildrenOf = filter.getQueryParam("directChildrenOf"); - String hierarchyFilter = filter.getQueryParam("hierarchyFilter"); - String offsetParam = filter.getQueryParam("offset"); - - if (!nullOrEmpty(directChildrenOf)) { - String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); - filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); - filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); - - condition += - " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; - } else if (Boolean.TRUE.toString().equals(hierarchyFilter)) { - // For hierarchy API, when directChildrenOf is null, show only root domains - condition += - " AND NOT EXISTS (SELECT 1 FROM entity_relationship er WHERE er.toId = domain_entity.id AND er.fromEntity = 'domain' AND er.toEntity = 'domain' AND er.relation = " - + Relationship.CONTAINS.ordinal() - + ")"; - } - - if (!nullOrEmpty(offsetParam) && Integer.parseInt(offsetParam) >= 0) { - return listAfter( - getTableName(), - filter.getQueryParams(), - condition, - limit, - Integer.parseInt(offsetParam)); - } - - return listAfter( - getTableName(), filter.getQueryParams(), condition, limit, afterName, afterId); - } - - @SqlQuery("SELECT json FROM domain_entity WHERE fqnHash LIKE :concatFqnhash ") - List getNestedDomains( - @BindConcat( - value = "concatFqnhash", - parts = {":fqnhash", ".%"}, - hash = true) - String fqnhash); - - @SqlQuery("SELECT COUNT(*) FROM domain_entity WHERE fqnHash LIKE :concatFqnhash ") - int countNestedDomains( - @BindConcat( - value = "concatFqnhash", - parts = {":fqnhash", ".%"}, - hash = true) - String fqnhash); - } - - interface DataProductDAO extends EntityDAO { - @Override - default String getTableName() { - return "data_product_entity"; - } - - @Override - default Class getEntityClass() { - return DataProduct.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default boolean supportsSoftDelete() { - return false; - } - } - - interface DataContractDAO extends EntityDAO { - @Override - default String getTableName() { - return "data_contract_entity"; - } - - @Override - default Class getEntityClass() { - return DataContract.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM data_contract_entity WHERE JSON_EXTRACT(json, '$.entity.id') = :entityId AND JSON_EXTRACT(json, '$.entity.type') = :entityType AND (JSON_EXTRACT(json, '$.deleted') IS NULL OR JSON_EXTRACT(json, '$.deleted') = false) LIMIT 1", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM data_contract_entity WHERE json#>>'{entity,id}' = :entityId AND json#>>'{entity,type}' = :entityType AND (json->>'deleted' IS NULL OR json->>'deleted' = 'false') LIMIT 1", - connectionType = POSTGRES) - String getContractByEntityId( - @Bind("entityId") String entityId, @Bind("entityType") String entityType); - } - - interface EventSubscriptionDAO extends EntityDAO { - @Override - default String getTableName() { - return "event_subscription_entity"; - } - - @Override - default Class getEntityClass() { - return EventSubscription.class; - } - - @SqlQuery("SELECT json FROM event_subscription_entity") - List listAllEventsSubscriptions(); - - @Override - default boolean supportsSoftDelete() { - return false; - } - - @SqlQuery("SELECT json FROM change_event_consumers where id = :id AND extension = :extension") - String getSubscriberExtension(@Bind("id") String id, @Bind("extension") String extension); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO change_event_consumers(id, extension, jsonSchema, json) " - + "VALUES (:id, :extension, :jsonSchema, :json)" - + "ON DUPLICATE KEY UPDATE json = :json, jsonSchema = :jsonSchema", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO change_event_consumers(id, extension, jsonSchema, json) " - + "VALUES (:id, :extension, :jsonSchema, (:json :: jsonb)) ON CONFLICT (id, extension) " - + "DO UPDATE SET json = EXCLUDED.json, jsonSchema = EXCLUDED.jsonSchema", - connectionType = POSTGRES) - void upsertSubscriberExtension( - @Bind("id") String id, - @Bind("extension") String extension, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO consumers_dlq(id, extension, json, source) " - + "VALUES (:id, :extension, :json, :source) " - + "ON DUPLICATE KEY UPDATE json = :json, source = :source", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO consumers_dlq(id, extension, json, source) " - + "VALUES (:id, :extension, (:json :: jsonb), :source) " - + "ON CONFLICT (id, extension) " - + "DO UPDATE SET json = EXCLUDED.json, source = EXCLUDED.source", - connectionType = POSTGRES) - void upsertFailedEvent( - @Bind("id") String id, - @Bind("extension") String extension, - @Bind("json") String json, - @Bind("source") String source); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO successful_sent_change_events (change_event_id, event_subscription_id, json, timestamp) " - + "VALUES (:change_event_id, :event_subscription_id, :json, :timestamp) " - + "ON DUPLICATE KEY UPDATE json = :json, timestamp = :timestamp", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO successful_sent_change_events (change_event_id, event_subscription_id, json, timestamp) " - + "VALUES (:change_event_id, :event_subscription_id, CAST(:json AS jsonb), :timestamp) " - + "ON CONFLICT (change_event_id, event_subscription_id) " - + "DO UPDATE SET json = EXCLUDED.json, timestamp = EXCLUDED.timestamp", - connectionType = POSTGRES) - void upsertSuccessfulChangeEvent( - @Bind("change_event_id") String changeEventId, - @Bind("event_subscription_id") String eventSubscriptionId, - @Bind("json") String json, - @Bind("timestamp") long timestamp); - - // Batch insert for successful events - reduces connection pool contention - // from N connections to 1 when processing multiple events - @Transaction - @ConnectionAwareSqlBatch( - value = - "INSERT INTO successful_sent_change_events (change_event_id, event_subscription_id, json, timestamp) " - + "VALUES (:change_event_id, :event_subscription_id, :json, :timestamp) " - + "ON DUPLICATE KEY UPDATE json = VALUES(json), timestamp = VALUES(timestamp)", - connectionType = MYSQL) - @ConnectionAwareSqlBatch( - value = - "INSERT INTO successful_sent_change_events (change_event_id, event_subscription_id, json, timestamp) " - + "VALUES (:change_event_id, :event_subscription_id, CAST(:json AS jsonb), :timestamp) " - + "ON CONFLICT (change_event_id, event_subscription_id) " - + "DO UPDATE SET json = EXCLUDED.json, timestamp = EXCLUDED.timestamp", - connectionType = POSTGRES) - void batchUpsertSuccessfulChangeEvents( - @Bind("change_event_id") List changeEventIds, - @Bind("event_subscription_id") List eventSubscriptionIds, - @Bind("json") List jsonList, - @Bind("timestamp") List timestamps); - - @SqlQuery( - "SELECT COUNT(*) FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId") - long getSuccessfulRecordCount(@Bind("eventSubscriptionId") String eventSubscriptionId); - - @SqlQuery( - "SELECT event_subscription_id FROM successful_sent_change_events " - + "GROUP BY event_subscription_id " - + "HAVING COUNT(*) >= :threshold") - List findSubscriptionsAboveThreshold(@Bind("threshold") int threshold); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId ORDER BY timestamp ASC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM successful_sent_change_events WHERE ctid IN (SELECT ctid FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId ORDER BY timestamp ASC LIMIT :limit)", - connectionType = POSTGRES) - void deleteOldRecords( - @Bind("eventSubscriptionId") String eventSubscriptionId, @Bind("limit") long limit); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM successful_sent_change_events " - + "WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM successful_sent_change_events " - + "WHERE ctid IN ( " - + " SELECT ctid FROM successful_sent_change_events " - + " WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit " - + ")", - connectionType = POSTGRES) - int deleteSuccessfulSentChangeEventsInBatches( - @Bind("cutoff") long cutoff, @Bind("limit") int limit); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM change_event " - + "WHERE eventTime < :cutoff ORDER BY eventTime LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM change_event " - + "WHERE ctid IN ( " - + " SELECT ctid FROM change_event " - + " WHERE eventTime < :cutoff ORDER BY eventTime LIMIT :limit " - + ")", - connectionType = POSTGRES) - int deleteChangeEventsInBatches(@Bind("cutoff") long cutoff, @Bind("limit") int limit); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM consumers_dlq " - + "WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM consumers_dlq " - + "WHERE ctid IN ( " - + " SELECT ctid FROM consumers_dlq " - + " WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit " - + ")", - connectionType = POSTGRES) - int deleteConsumersDlqInBatches(@Bind("cutoff") long cutoff, @Bind("limit") int limit); - - @SqlQuery( - "SELECT json FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") - List getSuccessfulChangeEventBySubscriptionId( - @Bind("eventSubscriptionId") String eventSubscriptionId, - @Bind("limit") int limit, - @Bind("paginationOffset") long paginationOffset); - - @SqlUpdate( - "DELETE FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId") - void deleteSuccessfulChangeEventBySubscriptionId( - @Bind("eventSubscriptionId") String eventSubscriptionId); - - @SqlUpdate("DELETE FROM consumers_dlq WHERE id = :eventSubscriptionId") - void deleteFailedRecordsBySubscriptionId( - @Bind("eventSubscriptionId") String eventSubscriptionId); - - @SqlUpdate("DELETE from change_event_consumers cec where id = :eventSubscriptionId;") - void deleteAlertMetrics(@Bind("eventSubscriptionId") String eventSubscriptionId); - - @ConnectionAwareSqlQuery( - value = - "SELECT COUNT(*) FROM ( " - + " SELECT json, 'FAILED' AS status, timestamp " - + " FROM consumers_dlq WHERE id = :id " - + " UNION ALL " - + " SELECT json, 'SUCCESSFUL' AS status, timestamp " - + " FROM successful_sent_change_events WHERE event_subscription_id = :id " - + ") AS combined_events", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT COUNT(*) FROM ( " - + " SELECT json, 'failed' AS status, timestamp " - + " FROM consumers_dlq WHERE id = :id " - + " UNION ALL " - + " SELECT json, 'successful' AS status, timestamp " - + " FROM successful_sent_change_events WHERE event_subscription_id = :id " - + ") AS combined_events", - connectionType = POSTGRES) - int countAllEventsWithStatuses(@Bind("id") String id); - - @SqlQuery("SELECT COUNT(*) FROM consumers_dlq WHERE id = :id") - int countFailedEventsById(@Bind("id") String id); - - @SqlQuery( - "SELECT COUNT(*) FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId") - int countSuccessfulEventsBySubscriptionId( - @Bind("eventSubscriptionId") String eventSubscriptionId); - } - - interface NotificationTemplateDAO extends EntityDAO { - @Override - default String getTableName() { - return "notification_template_entity"; - } - - @Override - default Class getEntityClass() { - return NotificationTemplate.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface ChartDAO extends EntityDAO { - @Override - default String getTableName() { - return "chart_entity"; - } - - @Override - default Class getEntityClass() { - return Chart.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface ApplicationDAO extends EntityDAO { - @Override - default String getTableName() { - return "installed_apps"; - } - - @Override - default Class getEntityClass() { - return App.class; - } - - @ConnectionAwareSqlQuery( - value = - "SELECT id, name, JSON_UNQUOTE(JSON_EXTRACT(json, '$.displayName')) as displayName from installed_apps", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT id, name, json ->> 'displayName' as displayName from installed_apps", - connectionType = POSTGRES) - @RegisterRowMapper(AppEntityReferenceMapper.class) - List listAppsRef(); - - class AppEntityReferenceMapper implements RowMapper { - @Override - public EntityReference map(ResultSet rs, StatementContext ctx) throws SQLException { - String fqn = rs.getString("name"); - String displayName = rs.getString("displayName"); - - return new EntityReference() - .withId(UUID.fromString(rs.getString("id"))) - .withName(fqn) - .withDisplayName(displayName) - .withFullyQualifiedName(fqn) - .withType(APPLICATION); - } - } - } - - interface ApplicationMarketPlaceDAO extends EntityDAO { - @Override - default String getTableName() { - return "apps_marketplace"; - } - - @Override - default Class getEntityClass() { - return AppMarketPlaceDefinition.class; - } - } - - interface MessagingServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "messaging_service_entity"; - } - - @Override - default Class getEntityClass() { - return MessagingService.class; - } - } - - interface MetricDAO extends EntityDAO { - @Override - default String getTableName() { - return "metric_entity"; - } - - @Override - default Class getEntityClass() { - return Metric.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @ConnectionAwareSqlQuery( - value = - "SELECT DISTINCT customUnitOfMeasurement AS customUnit " - + "FROM metric_entity " - + "WHERE customUnitOfMeasurement IS NOT NULL " - + "AND customUnitOfMeasurement != '' " - + "AND deleted = false " - + "ORDER BY customUnitOfMeasurement", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT DISTINCT customUnitOfMeasurement AS customUnit " - + "FROM metric_entity " - + "WHERE customUnitOfMeasurement IS NOT NULL " - + "AND customUnitOfMeasurement != '' " - + "AND deleted = false " - + "ORDER BY customUnitOfMeasurement", - connectionType = POSTGRES) - List getDistinctCustomUnitsOfMeasurement(); - } - - interface MlModelDAO extends EntityDAO { - @Override - default String getTableName() { - return "ml_model_entity"; - } - - @Override - default Class getEntityClass() { - return MlModel.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface GlossaryDAO extends EntityDAO { - @Override - default String getTableName() { - return "glossary_entity"; - } - - @Override - default Class getEntityClass() { - return Glossary.class; - } - } - - interface GlossaryTermDAO extends EntityDAO { - @Override - default String getTableName() { - return "glossary_term_entity"; - } - - @Override - default Class getEntityClass() { - return GlossaryTerm.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default int listCount(ListFilter filter) { - String condition = filter.getCondition(); - String directChildrenOf = filter.getQueryParam("directChildrenOf"); - - if (!nullOrEmpty(directChildrenOf)) { - String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); - filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); - filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); - - condition += - " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; - } - - return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), condition); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String condition = filter.getCondition(); - String directChildrenOf = filter.getQueryParam("directChildrenOf"); - - if (!nullOrEmpty(directChildrenOf)) { - String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); - filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); - filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); - - condition += - " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; - } - - return listBefore( - getTableName(), filter.getQueryParams(), condition, limit, beforeName, beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String condition = filter.getCondition(); - String directChildrenOf = filter.getQueryParam("directChildrenOf"); - - if (!nullOrEmpty(directChildrenOf)) { - String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); - filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); - filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); - - condition += - " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; - } - return listAfter( - getTableName(), filter.getQueryParams(), condition, limit, afterName, afterId); - } - - @SqlQuery("select json FROM glossary_term_entity where fqnhash LIKE :concatFqnhash ") - List getNestedTerms( - @BindConcat( - value = "concatFqnhash", - parts = {":fqnhash", ".%"}, - hash = true) - String fqnhash); - - @SqlQuery("SELECT COUNT(*) FROM glossary_term_entity WHERE fqnHash LIKE :concatFqnhash ") - int countNestedTerms( - @BindConcat( - value = "concatFqnhash", - parts = {":fqnhash", ".%"}, - hash = true) - String fqnhash); - - @SqlQuery( - "SELECT COUNT(*) FROM glossary_term_entity WHERE fqnHash LIKE :glossaryHash AND LOWER(name) = LOWER(:termName)") - int getGlossaryTermCountIgnoreCase( - @BindConcat( - value = "glossaryHash", - parts = {":fqnhash", ".%"}, - hash = true) - String fqnhash, - @Bind("termName") String termName); - - @SqlQuery( - "SELECT COUNT(*) FROM glossary_term_entity WHERE fqnHash LIKE :glossaryHash AND LOWER(name) = LOWER(:termName) AND id != :excludeId") - int getGlossaryTermCountIgnoreCaseExcludingId( - @BindConcat( - value = "glossaryHash", - parts = {":fqnhash", ".%"}, - hash = true) - String fqnhash, - @Bind("termName") String termName, - @Bind("excludeId") String excludeId); - - @SqlQuery( - "SELECT json FROM glossary_term_entity WHERE fqnHash LIKE :glossaryHash AND LOWER(name) = LOWER(:termName)") - String getGlossaryTermByNameAndGlossaryIgnoreCase( - @BindConcat( - value = "glossaryHash", - parts = {":fqnhash", ".%"}, - hash = true) - String fqnhash, - @Bind("termName") String termName); - - // Search glossary terms by name and displayName using LIKE queries - // The displayName column is a generated column added in migration 1.9.3 - // entityStatus filtering uses generated column added in migration 1.12.2 - @SqlQuery( - "SELECT json FROM glossary_term_entity WHERE deleted = FALSE " - + "AND fqnHash LIKE :parentHash " - + "AND (LOWER(name) LIKE LOWER(:searchTerm) " - + "OR LOWER(COALESCE(displayName, '')) LIKE LOWER(:searchTerm)) " - + " " - + "ORDER BY name " - + "LIMIT :limit OFFSET :offset") - List searchGlossaryTerms( - @Bind("parentHash") String parentHash, - @Bind("searchTerm") String searchTerm, - @Define("statusCondition") String statusCondition, - @Bind("limit") int limit, - @Bind("offset") int offset); - } - - interface IngestionPipelineDAO extends EntityDAO { - @Override - default String getTableName() { - return "ingestion_pipeline_entity"; - } - - @Override - default Class getEntityClass() { - return IngestionPipeline.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default int listCount(ListFilter filter) { - String condition = - "INNER JOIN entity_relationship ON ingestion_pipeline_entity.id = entity_relationship.toId"; - - if (filter.getQueryParam("pipelineType") != null) { - String pipelineTypeCondition = - String.format(" and %s", filter.getPipelineTypeCondition(null)); - condition += pipelineTypeCondition; - } - - if (filter.getQueryParam("applicationType") != null) { - String applicationTypeCondition = - String.format(" and %s", filter.getApplicationTypeCondition()); - condition += applicationTypeCondition; - } - - if (filter.getQueryParam("service") != null) { - String serviceCondition = String.format(" and %s", filter.getServiceCondition(null)); - condition += serviceCondition; - } - - if (filter.getQueryParam("provider") != null) { - String providerCondition = - String.format(" and %s", filter.getProviderCondition(getTableName())); - condition += providerCondition; - } - - Map bindMap = new HashMap<>(); - String serviceType = filter.getQueryParam("serviceType"); - String provider = filter.getQueryParam("provider"); - if (!nullOrEmpty(provider)) { - bindMap.put("provider", provider); - } - if (!nullOrEmpty(serviceType)) { - - condition = - String.format( - "%s WHERE entity_relationship.fromEntity = :serviceType and entity_relationship.relation = :relation", - condition); - bindMap.put("relation", CONTAINS.ordinal()); - return listIngestionPipelineCount(condition, bindMap, filter.getQueryParams()); - } - return EntityDAO.super.listCount(filter); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String condition = - "INNER JOIN entity_relationship ON ingestion_pipeline_entity.id = entity_relationship.toId"; - - if (filter.getQueryParam("pipelineType") != null) { - String pipelineTypeCondition = - String.format(" and %s", filter.getPipelineTypeCondition(null)); - condition += pipelineTypeCondition; - } - - if (filter.getQueryParam("applicationType") != null) { - String applicationTypeCondition = - String.format(" and %s", filter.getApplicationTypeCondition()); - condition += applicationTypeCondition; - } - - if (filter.getQueryParam("service") != null) { - String serviceCondition = String.format(" and %s", filter.getServiceCondition(null)); - condition += serviceCondition; - } - - if (filter.getQueryParam("provider") != null) { - String providerCondition = - String.format(" and %s", filter.getProviderCondition(getTableName())); - condition += providerCondition; - } - - Map bindMap = new HashMap<>(); - String serviceType = filter.getQueryParam("serviceType"); - String provider = filter.getQueryParam("provider"); - if (!nullOrEmpty(provider)) { - bindMap.put("provider", provider); - } - if (!nullOrEmpty(serviceType)) { - - condition = - String.format( - "%s WHERE entity_relationship.fromEntity = :serviceType and entity_relationship.relation = :relation and (ingestion_pipeline_entity.name > :afterName OR (ingestion_pipeline_entity.name = :afterName AND ingestion_pipeline_entity.id > :afterId)) order by ingestion_pipeline_entity.name ASC,ingestion_pipeline_entity.id ASC LIMIT :limit", - condition); - - bindMap.put("relation", CONTAINS.ordinal()); - bindMap.put("afterName", afterName); - bindMap.put("afterId", afterId); - bindMap.put("limit", limit); - return listAfterIngestionPipelineByserviceType(condition, bindMap, filter.getQueryParams()); - } - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String condition = - "INNER JOIN entity_relationship ON ingestion_pipeline_entity.id = entity_relationship.toId"; - - if (filter.getQueryParam("pipelineType") != null) { - String pipelineTypeCondition = - String.format(" and %s", filter.getPipelineTypeCondition(null)); - condition += pipelineTypeCondition; - } - - if (filter.getQueryParam("applicationType") != null) { - String applicationTypeCondition = - String.format(" and %s", filter.getApplicationTypeCondition()); - condition += applicationTypeCondition; - } - - if (filter.getQueryParam("service") != null) { - String serviceCondition = String.format(" and %s", filter.getServiceCondition(null)); - condition += serviceCondition; - } - - if (filter.getQueryParam("provider") != null) { - String providerCondition = - String.format(" and %s", filter.getProviderCondition(getTableName())); - condition += providerCondition; - } - - Map bindMap = new HashMap<>(); - String serviceType = filter.getQueryParam("serviceType"); - String provider = filter.getQueryParam("provider"); - if (!nullOrEmpty(provider)) { - bindMap.put("provider", provider); - } - if (!nullOrEmpty(serviceType)) { - condition = - String.format( - "%s WHERE entity_relationship.fromEntity = :serviceType and entity_relationship.relation = :relation and (ingestion_pipeline_entity.name < :beforeName OR (ingestion_pipeline_entity.name = :beforeName AND ingestion_pipeline_entity.id < :beforeId)) order by ingestion_pipeline_entity.name DESC, ingestion_pipeline_entity.id DESC LIMIT :limit", - condition); - - bindMap.put("relation", CONTAINS.ordinal()); - bindMap.put("beforeName", beforeName); - bindMap.put("beforeId", beforeId); - bindMap.put("limit", limit); - return listBeforeIngestionPipelineByserviceType( - condition, bindMap, filter.getQueryParams()); - } - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - - @SqlQuery("SELECT ingestion_pipeline_entity.json FROM ingestion_pipeline_entity ") - List listAfterIngestionPipelineByserviceType( - @Define("cond") String cond, - @BindMap Map bindings, - @BindMap Map params); - - @SqlQuery( - "SELECT json FROM (SELECT ingestion_pipeline_entity.name, ingestion_pipeline_entity.id, ingestion_pipeline_entity.json FROM ingestion_pipeline_entity ) last_rows_subquery ORDER BY last_rows_subquery.name,last_rows_subquery.id") - List listBeforeIngestionPipelineByserviceType( - @Define("cond") String cond, - @BindMap Map bindings, - @BindMap Map params); - - @SqlQuery("SELECT count(*) FROM ingestion_pipeline_entity ") - int listIngestionPipelineCount( - @Define("cond") String cond, - @BindMap Map bindings, - @BindMap Map params); - } - - interface PipelineServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "pipeline_service_entity"; - } - - @Override - default Class getEntityClass() { - return PipelineService.class; - } - } - - interface MlModelServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "mlmodel_service_entity"; - } - - @Override - default Class getEntityClass() { - return MlModelService.class; - } - } - - interface PolicyDAO extends EntityDAO { - @Override - default String getTableName() { - return "policy_entity"; - } - - @Override - default Class getEntityClass() { - return Policy.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface ReportDAO extends EntityDAO { - @Override - default String getTableName() { - return "report_entity"; - } - - @Override - default Class getEntityClass() { - return Report.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface TableDAO extends EntityDAO
{ - @Override - default String getTableName() { - return "table_entity"; - } - - @Override - default Class
getEntityClass() { - return Table.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @ConnectionAwareSqlQuery( - value = - "select JSON_EXTRACT(json, '$.fullyQualifiedName') from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "select json ->> 'fullyQualifiedName' from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')", - connectionType = POSTGRES) - List getBrokenTables(); - - @SqlUpdate( - value = - "delete from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')") - int removeBrokenTables(); - - @Override - default int listCount(ListFilter filter) { - String includeEmptyTestSuite = filter.getQueryParam("includeEmptyTestSuite"); - if (includeEmptyTestSuite != null && !Boolean.parseBoolean(includeEmptyTestSuite)) { - String condition = - String.format( - "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", - getTableName(), CONTAINS.ordinal(), Entity.TEST_SUITE); - String mySqlCondition = condition; - String postgresCondition = condition; - - mySqlCondition = - String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); - postgresCondition = - String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); - return listCount( - getTableName(), - getNameHashColumn(), - filter.getQueryParams(), - mySqlCondition, - postgresCondition); - } - - String condition = filter.getCondition(getTableName()); - return listCount( - getTableName(), getNameHashColumn(), filter.getQueryParams(), condition, condition); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String includeEmptyTestSuite = filter.getQueryParam("includeEmptyTestSuite"); - if (includeEmptyTestSuite != null && !Boolean.parseBoolean(includeEmptyTestSuite)) { - String condition = - String.format( - "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", - getTableName(), CONTAINS.ordinal(), Entity.TEST_SUITE); - String mySqlCondition = condition; - String postgresCondition = condition; - - mySqlCondition = - String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); - postgresCondition = - String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); - return listBefore( - getTableName(), - filter.getQueryParams(), - mySqlCondition, - postgresCondition, - limit, - beforeName, - beforeId); - } - String condition = filter.getCondition(getTableName()); - return listBefore( - getTableName(), - filter.getQueryParams(), - condition, - condition, - limit, - beforeName, - beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String includeEmptyTestSuite = filter.getQueryParam("includeEmptyTestSuite"); - if (includeEmptyTestSuite != null && !Boolean.parseBoolean(includeEmptyTestSuite)) { - String condition = - String.format( - "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", - getTableName(), CONTAINS.ordinal(), Entity.TEST_SUITE); - String mySqlCondition = condition; - String postgresCondition = condition; - - mySqlCondition = - String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); - postgresCondition = - String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); - return listAfter( - getTableName(), - filter.getQueryParams(), - mySqlCondition, - postgresCondition, - limit, - afterName, - afterId); - } - String condition = filter.getCondition(getTableName()); - return listAfter( - getTableName(), filter.getQueryParams(), condition, condition, limit, afterName, afterId); - } - } - - interface StoredProcedureDAO extends EntityDAO { - @Override - default String getTableName() { - return "stored_procedure_entity"; - } - - @Override - default Class getEntityClass() { - return StoredProcedure.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface QueryDAO extends EntityDAO { - @Override - default String getTableName() { - return "query_entity"; - } - - @Override - default Class getEntityClass() { - return Query.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default boolean supportsSoftDelete() { - return false; - } - - @Override - default int listCount(ListFilter filter) { - String entityId = filter.getQueryParam("entityId"); - String condition = - "INNER JOIN entity_relationship ON query_entity.id = entity_relationship.toId"; - Map bindMap = new HashMap<>(); - if (!nullOrEmpty(entityId)) { - condition = - String.format( - "%s WHERE entity_relationship.fromId = :id and entity_relationship.relation = :relation and entity_relationship.toEntity = :toEntityType", - condition); - bindMap.put("id", entityId); - bindMap.put("relation", MENTIONED_IN.ordinal()); - bindMap.put("toEntityType", QUERY); - return listQueryCount(condition, bindMap); - } - return EntityDAO.super.listCount(filter); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String entityId = filter.getQueryParam("entityId"); - String condition = - "INNER JOIN entity_relationship ON query_entity.id = entity_relationship.toId"; - Map bindMap = new HashMap<>(); - if (!nullOrEmpty(entityId)) { - condition = - String.format( - "%s WHERE entity_relationship.fromId = :entityId and entity_relationship.relation = :relation and entity_relationship.toEntity = :toEntity and (query_entity.name < :beforeName OR (query_entity.name = :beforeName AND query_entity.id < :beforeId)) order by query_entity.name DESC, query_entity.id DESC LIMIT :limit", - condition); - bindMap.put("entityId", entityId); - bindMap.put("relation", MENTIONED_IN.ordinal()); - bindMap.put("toEntity", QUERY); - bindMap.put("beforeName", beforeName); - bindMap.put("beforeId", beforeId); - bindMap.put("limit", limit); - return listBeforeQueriesByEntityId(condition, bindMap); - } - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String entityId = filter.getQueryParam("entityId"); - String condition = - "INNER JOIN entity_relationship ON query_entity.id = entity_relationship.toId"; - Map bindMap = new HashMap<>(); - if (!nullOrEmpty(entityId)) { - condition = - String.format( - "%s WHERE entity_relationship.fromId = :entityId and entity_relationship.relation = :relation and entity_relationship.toEntity = :toEntity and (query_entity.name > :afterName OR (query_entity.name = :afterName AND query_entity.name > :afterId)) order by query_entity.name ASC,query_entity.id ASC LIMIT :limit", - condition); - - bindMap.put("entityId", entityId); - bindMap.put("relation", MENTIONED_IN.ordinal()); - bindMap.put("toEntity", QUERY); - bindMap.put("afterName", afterName); - bindMap.put("afterId", afterId); - bindMap.put("limit", limit); - return listAfterQueriesByEntityId(condition, bindMap); - } - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - - @SqlQuery("SELECT query_entity.json FROM query_entity ") - List listAfterQueriesByEntityId( - @Define("cond") String cond, @BindMap Map bindings); - - @SqlQuery( - "SELECT json FROM (SELECT query_entity.name, query_entity.id, query_entity.json FROM query_entity ) last_rows_subquery ORDER BY name,id") - List listBeforeQueriesByEntityId( - @Define("cond") String cond, @BindMap Map bindings); - - @SqlQuery("SELECT count(*) FROM query_entity ") - int listQueryCount(@Define("cond") String cond, @BindMap Map bindings); - } - - interface PipelineDAO extends EntityDAO { - @Override - default String getTableName() { - return "pipeline_entity"; - } - - @Override - default Class getEntityClass() { - return Pipeline.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String status = filter.getQueryParam("status"); - if (status != null && !status.isEmpty()) { - // Remove status from filter to avoid SQL error - Map params = new HashMap<>(filter.getQueryParams()); - params.remove("status"); - ListFilter cleanFilter = new ListFilter(filter.getInclude()); - params.forEach(cleanFilter::addQueryParam); - - // Build condition with status JOIN - String condition = cleanFilter.getCondition(); - String statusCondition = - buildStatusJoinCondition(getTableName(), condition, status, beforeName, beforeId, true); - return listBeforeWithStatus( - statusCondition, getBindMap(cleanFilter, status, limit, beforeName, beforeId)); - } - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String status = filter.getQueryParam("status"); - if (status != null && !status.isEmpty()) { - // Remove status from filter to avoid SQL error - Map params = new HashMap<>(filter.getQueryParams()); - params.remove("status"); - ListFilter cleanFilter = new ListFilter(filter.getInclude()); - params.forEach(cleanFilter::addQueryParam); - - // Build condition with status JOIN - String condition = cleanFilter.getCondition(); - String statusCondition = - buildStatusJoinCondition(getTableName(), condition, status, afterName, afterId, false); - return listAfterWithStatus( - statusCondition, getBindMap(cleanFilter, status, limit, afterName, afterId)); - } - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - - @Override - default int listCount(ListFilter filter) { - String status = filter.getQueryParam("status"); - if (status != null && !status.isEmpty()) { - // Remove status from filter to avoid SQL error - Map params = new HashMap<>(filter.getQueryParams()); - params.remove("status"); - ListFilter cleanFilter = new ListFilter(filter.getInclude()); - params.forEach(cleanFilter::addQueryParam); - - // Build condition with status JOIN - String condition = cleanFilter.getCondition(); - String statusCondition = buildStatusCountCondition(getTableName(), condition, status); - return listCountWithStatus(statusCondition, getBindMap(cleanFilter, status, 0, null, null)); - } - return EntityDAO.super.listCount(filter); - } - - default String buildStatusJoinCondition( - String tableName, - String baseCondition, - String status, - String name, - String id, - boolean isBefore) { - String orderDirection = isBefore ? "DESC" : "ASC"; - String nameComparison = isBefore ? "<" : ">"; - String idComparison = isBefore ? "<" : ">"; - - return String.format( - "INNER JOIN (" - + " SELECT entityFQNHash, JSON_UNQUOTE(JSON_EXTRACT(json, '$.executionStatus')) as execStatus " - + " FROM entity_extension_time_series " - + " WHERE extension = 'pipeline.pipelineStatus' " - + " AND timestamp = (SELECT MAX(timestamp) FROM entity_extension_time_series eets2 " - + " WHERE eets2.entityFQNHash = entity_extension_time_series.entityFQNHash " - + " AND eets2.extension = 'pipeline.pipelineStatus') " - + ") latest_status ON %s.fqnHash = latest_status.entityFQNHash " - + "%s AND latest_status.execStatus = :status " - + "AND (%s.name %s :beforeAfterName OR (%s.name = :beforeAfterName AND %s.id %s :beforeAfterId)) " - + "ORDER BY %s.name %s, %s.id %s LIMIT :limit", - tableName, - baseCondition, - tableName, - nameComparison, - tableName, - tableName, - idComparison, - tableName, - orderDirection, - tableName, - orderDirection); - } - - default String buildStatusCountCondition( - String tableName, String baseCondition, String status) { - return String.format( - "INNER JOIN (" - + " SELECT entityFQNHash, JSON_UNQUOTE(JSON_EXTRACT(json, '$.executionStatus')) as execStatus " - + " FROM entity_extension_time_series " - + " WHERE extension = 'pipeline.pipelineStatus' " - + " AND timestamp = (SELECT MAX(timestamp) FROM entity_extension_time_series eets2 " - + " WHERE eets2.entityFQNHash = entity_extension_time_series.entityFQNHash " - + " AND eets2.extension = 'pipeline.pipelineStatus') " - + ") latest_status ON %s.fqnHash = latest_status.entityFQNHash " - + "%s AND latest_status.execStatus = :status", - tableName, baseCondition); - } - - default Map getBindMap( - ListFilter filter, String status, int limit, String name, String id) { - Map bindMap = new HashMap<>(); - if (status != null) { - bindMap.put("status", status); - } - if (limit > 0) { - bindMap.put("limit", limit); - } - if (name != null) { - bindMap.put("beforeAfterName", name); - } - if (id != null) { - bindMap.put("beforeAfterId", id); - } - // Add filter params - bindMap.putAll(filter.getQueryParams()); - return bindMap; - } - - @SqlQuery("SELECT json FROM pipeline_entity ") - List listAfterWithStatus( - @Define("cond") String cond, @BindMap Map bindings); - - @SqlQuery( - "SELECT json FROM (SELECT name, id, json FROM pipeline_entity ) last_rows_subquery ORDER BY name, id") - List listBeforeWithStatus( - @Define("cond") String cond, @BindMap Map bindings); - - @SqlQuery("SELECT count(*) FROM pipeline_entity ") - int listCountWithStatus(@Define("cond") String cond, @BindMap Map bindings); - } - - interface ClassificationDAO extends EntityDAO { - @Override - default String getTableName() { - return "classification"; - } - - @Override - default Class getEntityClass() { - return Classification.class; - } - - // Much more efficient: Use IN clause with proper index usage - @SqlQuery( - "SELECT classificationHash, COUNT(*) AS termCount " - + "FROM tag " - + "WHERE classificationHash IN () " - + "AND deleted = FALSE " - + "GROUP BY classificationHash") - @RegisterRowMapper(TermCountMapper.class) - List> bulkGetTermCounts(@BindList("hashArray") List hashArray); - - class TermCountMapper implements RowMapper> { - @Override - public Pair map(ResultSet rs, StatementContext ctx) throws SQLException { - return Pair.of(rs.getString("classificationHash"), rs.getInt("termCount")); - } - } - } - - interface TagDAO extends EntityDAO { - @Override - default String getTableName() { - return "tag"; - } - - @Override - default Class getEntityClass() { - return Tag.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - private Pair buildTagQueryConditions(ListFilter filter) { - String parent = filter.getQueryParam("parent"); - boolean disabled = Boolean.parseBoolean(filter.getQueryParam("classification.disabled")); - - String baseJoin = - String.format( - "INNER JOIN entity_relationship er ON tag.id=er.toId AND er.relation=%s AND er.fromEntity='%s' " - + "INNER JOIN classification c ON er.fromId=c.id", - CONTAINS.ordinal(), Entity.CLASSIFICATION); - - StringBuilder mySqlCondition = new StringBuilder(baseJoin); - StringBuilder postgresCondition = new StringBuilder(baseJoin); - - if (parent != null) { - String parentFqnHash = FullyQualifiedName.buildHash(parent); - filter.queryParams.put("parentFqnPrefix", parentFqnHash + ".%"); - mySqlCondition.append(" AND tag.fqnHash LIKE :parentFqnPrefix"); - postgresCondition.append(" AND tag.fqnHash LIKE :parentFqnPrefix"); - } - - if (disabled) { - mySqlCondition.append( - " AND (JSON_EXTRACT(c.json, '$.disabled') = TRUE OR JSON_EXTRACT(tag.json, '$.disabled') = TRUE)"); - postgresCondition.append( - " AND (COALESCE((c.json->>'disabled')::boolean, FALSE) = TRUE OR COALESCE((tag.json->>'disabled')::boolean, FALSE) = TRUE)"); - } else if (filter.getQueryParam("classification.disabled") != null) { - mySqlCondition.append( - " AND (JSON_EXTRACT(c.json, '$.disabled') IS NULL OR JSON_EXTRACT(c.json, '$.disabled') = FALSE) AND (JSON_EXTRACT(tag.json, '$.disabled') IS NULL OR JSON_EXTRACT(tag.json, '$.disabled') = FALSE)"); - postgresCondition.append( - " AND COALESCE((c.json->>'disabled')::boolean, FALSE) = FALSE AND COALESCE((tag.json->>'disabled')::boolean, FALSE) = FALSE"); - } - - String tagCondition = filter.getCondition("tag"); - if (!tagCondition.isEmpty()) { - mySqlCondition.append(" ").append(tagCondition); - postgresCondition.append(" ").append(tagCondition); - } - - return Pair.of(mySqlCondition.toString(), postgresCondition.toString()); - } - - @Override - default int listCount(ListFilter filter) { - Pair conditions = buildTagQueryConditions(filter); - return listCount( - getTableName(), - getNameHashColumn(), - filter.getQueryParams(), - conditions.getLeft(), - conditions.getRight()); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - Pair conditions = buildTagQueryConditions(filter); - return listBefore( - getTableName(), - filter.getQueryParams(), - conditions.getLeft(), - conditions.getRight(), - limit, - beforeName, - beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - Pair conditions = buildTagQueryConditions(filter); - return listAfter( - getTableName(), - filter.getQueryParams(), - conditions.getLeft(), - conditions.getRight(), - limit, - afterName, - afterId); - } - - @SqlQuery("select json FROM tag where fqnhash LIKE :concatFqnhash") - List getTagsStartingWithPrefix( - @BindConcat( - value = "concatFqnhash", - parts = {":fqnhash", ".%"}, - hash = true) - String fqnhash); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE tag SET json = JSON_SET(json, '$.recognizers', CAST(:recognizers AS JSON)) " - + "WHERE JSON_EXTRACT(json, '$.fullyQualifiedName') = :tagFqn", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE tag SET json = json::jsonb || json_build_object(" - + "'recognizers', :recognizers::jsonb " - + ")::jsonb WHERE json->>'fullyQualifiedName' = :tagFqn", - connectionType = POSTGRES) - void patchRecognizers(@Bind("tagFqn") String tagFqn, @Bind("recognizers") String recognizers); - } - - @RegisterRowMapper(TagLabelMapper.class) - interface TagUsageDAO { - @ConnectionAwareSqlUpdate( - value = - "INSERT IGNORE INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedBy, metadata) VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :reason, :appliedBy, :metadata)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedBy, metadata) VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :reason, :appliedBy, :metadata :: jsonb) ON CONFLICT (source, tagFQNHash, targetFQNHash) DO NOTHING", - connectionType = POSTGRES) - void applyTag( - @Bind("source") int source, - @Bind("tagFQN") String tagFQN, - @BindFQN("tagFQNHash") String tagFQNHash, - @BindFQN("targetFQNHash") String targetFQNHash, - @Bind("labelType") int labelType, - @Bind("state") int state, - @Bind("reason") String reason, - @Bind("appliedBy") String appliedBy, - @Bind("metadata") String metadata); - - default void applyTag( - int source, - String tagFQN, - String tagFQNHash, - String targetFQNHash, - int labelType, - int state, - String reason, - String appliedBy, - TagLabelMetadata metadata) { - this.applyTag( - source, - tagFQN, - tagFQNHash, - targetFQNHash, - labelType, - state, - reason, - appliedBy, - JsonUtils.pojoToJson(metadata)); - } - - default void applyTag( - int source, - String tagFQN, - String tagFQNHash, - String targetFQNHash, - int labelType, - int state, - String reason, - String appliedBy) { - this.applyTag( - source, - tagFQN, - tagFQNHash, - targetFQNHash, - labelType, - state, - reason, - appliedBy, - (String) null); - } - - default List getTags(String targetFQN) { - List tags = getTagsInternal(targetFQN); - tags.forEach(TagLabelUtil::applyTagCommonFieldsGracefully); - return tags; - } - - default Map> getTagsByPrefix( - String targetFQNPrefix, String postfix, boolean requiresFqnHash) { - String targetFQNPrefixHash = - requiresFqnHash ? FullyQualifiedName.buildHash(targetFQNPrefix) : targetFQNPrefix; - Map> resultSet = new LinkedHashMap<>(); - List> tags = getTagsInternalByPrefix(targetFQNPrefixHash, postfix); - tags.forEach( - pair -> { - String targetHash = pair.getLeft(); - TagLabel tagLabel = pair.getRight(); - List listOfTarget = new ArrayList<>(); - if (resultSet.containsKey(targetHash)) { - listOfTarget = resultSet.get(targetHash); - listOfTarget.add(tagLabel); - } else { - listOfTarget.add(tagLabel); - } - resultSet.put(targetHash, listOfTarget); - }); - return resultSet; - } - - @SqlQuery( - "SELECT source, tagFQN, labelType, state, reason, appliedAt, appliedBy, metadata FROM tag_usage WHERE targetFQNHash = :targetFQNHash ORDER BY tagFQN") - List getTagsInternal(@BindFQN("targetFQNHash") String targetFQNHash); - - @SqlQuery( - "SELECT targetFQNHash, source, tagFQN, labelType, state, reason, appliedAt, appliedBy, metadata " - + "FROM tag_usage " - + "WHERE targetFQNHash IN () " - + "ORDER BY targetFQNHash, tagFQN") - @UseRowMapper(TagLabelWithFQNHashMapper.class) - List getTagsInternalBatch( - @BindListFQN("targetFQNHashes") List targetFQNHashes); - - @SqlQuery( - "SELECT targetFQNHash, source, tagFQN, labelType, state, reason, appliedAt, appliedBy, metadata " - + "FROM tag_usage " - + "WHERE source = :source " - + "AND targetFQNHash IN () " - + "AND tagFQNHash LIKE :tagFQNHashPrefix " - + "ORDER BY targetFQNHash, tagFQN") - @UseRowMapper(TagLabelWithFQNHashMapper.class) - List getCertTagsInternalBatch( - @Bind("source") int source, - @BindListFQN("targetFQNHashes") List targetFQNHashes, - @Bind("tagFQNHashPrefix") String tagFQNHashPrefix); - - /** - * Batch fetch derived tags for multiple glossary term FQNs. Returns a map from glossary term - * FQN to its derived tags (tags that target that glossary term). - */ - default Map> getDerivedTagsBatch(List glossaryTermFqns) { - if (glossaryTermFqns == null || glossaryTermFqns.isEmpty()) { - return Collections.emptyMap(); - } - List tagUsages = getTagsInternalBatch(glossaryTermFqns); - Map> result = new HashMap<>(); - - for (TagLabelWithFQNHash usage : tagUsages) { - String targetFqn = usage.getTargetFQNHash(); - TagLabel tagLabel = - new TagLabel() - .withSource(TagLabel.TagSource.values()[usage.getSource()]) - .withTagFQN(usage.getTagFQN()) - .withLabelType(TagLabel.LabelType.DERIVED) - .withState(TagLabel.State.values()[usage.getState()]) - .withReason(usage.getReason()) - .withAppliedAt(usage.toTagLabel().getAppliedAt()); - if (usage.getReason() != null) { - tagLabel.withReason(usage.getReason()); - } - TagLabelUtil.applyTagCommonFieldsGracefully(tagLabel); - result.computeIfAbsent(targetFqn, k -> new ArrayList<>()).add(tagLabel); - } - return result; - } - - @ConnectionAwareSqlQuery( - value = - "SELECT tu.source, tu.tagFQN, tu.labelType, tu.targetFQNHash, tu.state, tu.reason, tu.appliedAt, tu.appliedBy, tu.metadata, " - + "CASE " - + " WHEN tu.source = 1 THEN gterm.json " - + " WHEN tu.source = 0 THEN ta.json " - + "END as json " - + "FROM tag_usage tu " - + "LEFT JOIN glossary_term_entity gterm ON tu.source = 1 AND gterm.fqnHash = tu.tagFQNHash " - + "LEFT JOIN tag ta ON tu.source = 0 AND ta.fqnHash = tu.tagFQNHash " - + "WHERE tu.targetfqnhash_lower LIKE LOWER(:targetFQNHash)", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT tu.source, tu.tagFQN, tu.labelType, tu.targetFQNHash, tu.state, tu.reason, tu.appliedAt, tu.appliedBy, tu.metadata, " - + "CASE " - + " WHEN tu.source = 1 THEN gterm.json " - + " WHEN tu.source = 0 THEN ta.json " - + "END as json " - + "FROM tag_usage tu " - + "LEFT JOIN glossary_term_entity gterm ON tu.source = 1 AND gterm.fqnHash = tu.tagFQNHash " - + "LEFT JOIN tag ta ON tu.source = 0 AND ta.fqnHash = tu.tagFQNHash " - + "WHERE tu.targetfqnhash_lower LIKE LOWER(:targetFQNHash)", - connectionType = POSTGRES) - @RegisterRowMapper(TagLabelRowMapperWithTargetFqnHash.class) - List> getTagsInternalByPrefix( - @BindConcat( - value = "targetFQNHash", - parts = {":targetFQNHashPrefix", ":postfix"}) - String... targetFQNHash); - - @SqlQuery( - "SELECT tu.source, tu.tagFQN, tu.labelType, tu.targetFQNHash, tu.state, tu.reason, tu.appliedAt, tu.appliedBy, tu.metadata, " - + "CASE " - + " WHEN tu.source = 1 THEN gterm.json " - + " WHEN tu.source = 0 THEN ta.json " - + "END as json " - + "FROM tag_usage tu " - + "LEFT JOIN glossary_term_entity gterm ON tu.source = 1 AND gterm.fqnHash = tu.tagFQNHash " - + "LEFT JOIN tag ta ON tu.source = 0 AND ta.fqnHash = tu.tagFQNHash " - + "WHERE tu.targetFQNHash IN ()") - @RegisterRowMapper(TagLabelRowMapperWithTargetFqnHash.class) - List> getTagsInternalByTargetHashes( - @BindList("targetFQNHashes") List targetFQNHashes); - - int TAG_BATCH_CHUNK_SIZE = 1000; - - default Map> getTagsByTargetFQNHashes(List targetFQNHashes) { - Map> resultSet = new LinkedHashMap<>(); - if (targetFQNHashes == null || targetFQNHashes.isEmpty()) { - return resultSet; - } - for (int i = 0; i < targetFQNHashes.size(); i += TAG_BATCH_CHUNK_SIZE) { - List chunk = - targetFQNHashes.subList(i, Math.min(i + TAG_BATCH_CHUNK_SIZE, targetFQNHashes.size())); - for (Pair pair : getTagsInternalByTargetHashes(chunk)) { - resultSet.computeIfAbsent(pair.getLeft(), k -> new ArrayList<>()).add(pair.getRight()); - } - } - return resultSet; - } - - @SqlQuery("SELECT * FROM tag_usage") - @Deprecated(since = "Release 1.1") - @RegisterRowMapper(TagLabelMapperMigration.class) - List listAll(); - - @SqlQuery( - "SELECT COUNT(*) FROM tag_usage " - + "WHERE (tagFQNHash LIKE :concatTagFQNHash OR tagFQNHash = :tagFqnHash) " - + "AND source = :source") - int getTagCount( - @Bind("source") int source, - @BindConcat( - value = "concatTagFQNHash", - original = "tagFqnHash", - parts = {":tagFqnHash", ".%"}, - hash = true) - String tagFqnHash); - - /** - * Get tag usage counts for multiple tags. - * This method retrieves counts for exact tag matches and their children in one query. - */ - @SqlQuery( - "SELECT tagFQN, count FROM (" - + " SELECT ? as tagFQN, COUNT(DISTINCT targetFQNHash) as count " - + " FROM tag_usage " - + " WHERE source = ? AND (tagFQNHash = MD5(?) OR tagFQNHash LIKE CONCAT(MD5(?), '.%'))" - + ") t WHERE tagFQN IN ()") - @RegisterRowMapper(TagCountMapper.class) - @Deprecated - List> getTagCountsBulkComplex( - @Bind("tagFQN") String sampleTagFQN, - @Bind("source") int source, - @Bind("tagFQNHash") String tagFQNHash, - @Bind("tagFQNHashPrefix") String tagFQNHashPrefix, - @BindList("tagFQNs") List tagFQNs); - - default Map getTagCountsBulk(int source, List tagFQNs) { - if (tagFQNs == null || tagFQNs.isEmpty()) { - return Collections.emptyMap(); - } - - Map resultMap = new HashMap<>(); - - // Process tags in batches to create a single efficient query - // We'll use a UNION ALL approach which is more compatible with JDBI - StringBuilder queryBuilder = new StringBuilder(); - List params = new ArrayList<>(); - - for (int i = 0; i < tagFQNs.size(); i++) { - if (i > 0) { - queryBuilder.append(" UNION ALL "); - } - queryBuilder.append( - "SELECT ? as tagFQN, COUNT(DISTINCT targetFQNHash) as count " - + "FROM tag_usage " - + "WHERE source = ? AND (tagFQNHash = MD5(?) OR tagFQNHash LIKE CONCAT(MD5(?), '.%'))"); - params.add(tagFQNs.get(i)); // tagFQN for selection - params.add(String.valueOf(source)); // source - params.add(tagFQNs.get(i)); // tagFQN for MD5 - params.add(tagFQNs.get(i)); // tagFQN for LIKE - } - - // For now, fall back to individual queries until we have a better solution - // This ensures correctness while we work on optimization - for (String tagFQN : tagFQNs) { - int count = getTagCount(source, tagFQN); - resultMap.put(tagFQN, count); - } - - return resultMap; - } - - @SqlUpdate("DELETE FROM tag_usage where targetFQNHash = :targetFQNHash") - void deleteTagsByTarget(@BindFQN("targetFQNHash") String targetFQNHash); - - @SqlUpdate( - "DELETE FROM tag_usage WHERE source = :source AND tagFQN LIKE :tagFQNPrefix AND targetFQNHash = :targetFQNHash") - void deleteTagsByPrefixAndTarget( - @Bind("source") int source, - @Bind("tagFQNPrefix") String tagFQNPrefix, - @BindFQN("targetFQNHash") String targetFQNHash); - - @SqlUpdate("DELETE FROM tag_usage WHERE targetFQNHash IN ()") - void deleteTagsByTargets(@BindListFQN("targetFQNHashes") List targetFQNs); - - @SqlUpdate( - "DELETE FROM tag_usage WHERE source = :source AND tagFQN LIKE :tagFQNPrefix AND targetFQNHash IN ()") - void deleteTagsByPrefixAndTargets( - @Bind("source") int source, - @Bind("tagFQNPrefix") String tagFQNPrefix, - @BindListFQN("targetFQNHashes") List targetFQNHashes); - - @SqlUpdate( - "DELETE FROM tag_usage where tagFQNHash = :tagFqnHash AND targetFQNHash LIKE :targetFQNHash") - void deleteTagsByTagAndTargetEntity( - @BindFQN("tagFqnHash") String tagFqnHash, - @BindConcat( - value = "targetFQNHash", - parts = {":targetFQNHashPrefix", "%"}, - hash = true) - String targetFQNHashPrefix); - - @SqlUpdate("DELETE FROM tag_usage where tagFQNHash = :tagFQNHash AND source = :source") - void deleteTagLabels(@Bind("source") int source, @BindFQN("tagFQNHash") String tagFQNHash); - - @ConnectionAwareSqlUpdate( - value = "DELETE FROM tag_usage where tagFQNHash = :tagFQNHash ORDER BY tagFQN", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "DELETE FROM tag_usage where tagFQNHash = :tagFQNHash", - connectionType = POSTGRES) - void deleteTagLabelsByFqn(@BindFQN("tagFQNHash") String tagFQNHash); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM tag_usage where targetFQNHash = :targetFQNHash OR targetFQNHash LIKE :concatTargetFQNHash ORDER BY tagFQN", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM tag_usage where targetFQNHash = :targetFQNHash OR targetFQNHash LIKE :concatTargetFQNHash", - connectionType = POSTGRES) - void deleteTagLabelsByTargetPrefix( - @BindConcat( - value = "concatTargetFQNHash", - original = "targetFQNHash", - parts = {":targetFQNHashPrefix", ".%"}, - hash = true) - String targetFQNHashPrefix); - - @Deprecated(since = "Release 1.1") - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, targetFQN)" - + "VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :targetFQN) " - + "ON DUPLICATE KEY UPDATE tagFQNHash = :tagFQNHash, targetFQNHash = :targetFQNHash", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, targetFQN) " - + "VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :targetFQN) " - + "ON CONFLICT (source, tagFQN, targetFQN) " - + "DO UPDATE SET tagFQNHash = EXCLUDED.tagFQNHash, targetFQNHash = EXCLUDED.targetFQNHash", - connectionType = POSTGRES) - void upsertFQNHash( - @Bind("source") int source, - @Bind("tagFQN") String tagFQN, - @Bind("tagFQNHash") String tagFQNHash, - @Bind("targetFQNHash") String targetFQNHash, - @Bind("labelType") int labelType, - @Bind("state") int state, - @Bind("targetFQN") String targetFQN); - - /** Update all the tagFQN starting with oldPrefix to start with newPrefix due to tag or glossary name change */ - default void updateTagPrefix(int source, String oldPrefix, String newPrefix) { - String update = - String.format( - "UPDATE tag_usage SET tagFQN = REPLACE(tagFQN, '%s.', '%s.'), tagFQNHash = REPLACE(tagFQNHash, '%s.', '%s.') WHERE source = %s AND tagFQNHash LIKE '%s.%%'", - escapeApostrophe(oldPrefix), - escapeApostrophe(newPrefix), - FullyQualifiedName.buildHash(oldPrefix), - FullyQualifiedName.buildHash(newPrefix), - source, - FullyQualifiedName.buildHash(oldPrefix)); - updateTagPrefixInternal(update); - } - - default void updateTargetFQNHashPrefix( - int source, String oldTargetFQNHashPrefix, String newTargetFQNHashPrefix) { - String update = - String.format( - "UPDATE tag_usage SET targetFQNHash = REPLACE(targetFQNHash, '%s.', '%s.') WHERE source = %s AND targetFQNHash LIKE '%s.%%'", - FullyQualifiedName.buildHash(oldTargetFQNHashPrefix), - FullyQualifiedName.buildHash(newTargetFQNHashPrefix), - source, - FullyQualifiedName.buildHash(oldTargetFQNHashPrefix)); - updateTagPrefixInternal(update); - } - - default void rename(int source, String oldFQN, String newFQN) { - renameInternal(source, oldFQN, newFQN, newFQN); // First rename tagFQN from oldFQN to newFQN - updateTagPrefix( - source, oldFQN, - newFQN); // Rename all the tagFQN prefixes starting with the oldFQN to newFQN - } - - default void renameByTargetFQNHash( - int source, String oldTargetFQNHash, String newTargetFQNHash) { - updateTargetFQNHashPrefix( - source, - oldTargetFQNHash, - newTargetFQNHash); // Rename all the targetFQN prefixes starting with the oldFQN to newFQN - } - - /** Rename the tagFQN */ - @SqlUpdate( - "Update tag_usage set tagFQN = :newFQN, tagFQNHash = :newFQNHash WHERE source = :source AND tagFQNHash = :oldFQNHash") - void renameInternal( - @Bind("source") int source, - @BindFQN("oldFQNHash") String oldFQNHash, - @Bind("newFQN") String newFQN, - @BindFQN("newFQNHash") String newFQNHash); - - @SqlUpdate( - "UPDATE tag_usage SET targetFQNHash = :newTargetFQNHash WHERE targetFQNHash = :oldTargetFQNHash") - void updateTargetFQNHash( - @BindFQN("oldTargetFQNHash") String oldTargetFQNHash, - @BindFQN("newTargetFQNHash") String newTargetFQNHash); - - @SqlUpdate("") - void updateTagPrefixInternal(@Define("update") String update); - - @SqlQuery("select targetFQNHash FROM tag_usage where tagFQNHash = :tagFQNHash") - @RegisterRowMapper(TagLabelMapper.class) - List getTargetFQNHashForTag(@BindFQN("tagFQNHash") String tagFQNHash); - - @SqlQuery("select targetFQNHash FROM tag_usage where tagFQNHash LIKE :tagFQNHash") - @RegisterRowMapper(TagLabelMapper.class) - List getTargetFQNHashForTagPrefix( - @BindConcat( - value = "tagFQNHash", - parts = {":tagFQNHashPrefix", ".%"}, - hash = true) - String tagFQNHashPrefix); - - class TagLabelMapper implements RowMapper { - @Override - public TagLabel map(ResultSet r, StatementContext ctx) throws SQLException { - TagLabelMetadata metadata = null; - try { - metadata = JsonUtils.readValue(r.getString("metadata"), TagLabelMetadata.class); - } catch (Exception e) { - // Ignore unknown fields from future schema versions — metadata is best-effort - } - return new TagLabel() - .withSource(TagLabel.TagSource.values()[r.getInt("source")]) - .withLabelType(TagLabel.LabelType.values()[r.getInt("labelType")]) - .withState(TagLabel.State.values()[r.getInt("state")]) - .withTagFQN(r.getString("tagFQN")) - .withReason(r.getString("reason")) - .withAppliedAt(r.getTimestamp("appliedAt")) - .withAppliedBy(r.getString("appliedBy")) - .withMetadata(metadata); - } - } - - class TagCountMapper implements RowMapper> { - @Override - public Map.Entry map(ResultSet r, StatementContext ctx) throws SQLException { - String tagFQN = r.getString("tagFQN"); - int count = r.getInt("count"); - return new AbstractMap.SimpleEntry<>(tagFQN, count); - } - } - - class TagLabelRowMapperWithTargetFqnHash implements RowMapper> { - @Override - public Pair map(ResultSet r, StatementContext ctx) throws SQLException { - TagLabel label = - new TagLabel() - .withSource(TagLabel.TagSource.values()[r.getInt("source")]) - .withLabelType(TagLabel.LabelType.values()[r.getInt("labelType")]) - .withState(TagLabel.State.values()[r.getInt("state")]) - .withTagFQN(r.getString("tagFQN")) - .withReason(r.getString("reason")) - .withAppliedAt(r.getTimestamp("appliedAt")) - .withAppliedBy(r.getString("appliedBy")) - .withMetadata(JsonUtils.readValue(r.getString("metadata"), TagLabelMetadata.class)); - TagLabel.TagSource source = TagLabel.TagSource.values()[r.getInt("source")]; - if (source == TagLabel.TagSource.CLASSIFICATION) { - Tag tag = JsonUtils.readValue(r.getString("json"), Tag.class); - label.setName(tag.getName()); - label.setDisplayName(tag.getDisplayName()); - label.setDescription(tag.getDescription()); - label.setStyle(tag.getStyle()); - } else if (source == TagLabel.TagSource.GLOSSARY) { - GlossaryTerm glossaryTerm = JsonUtils.readValue(r.getString("json"), GlossaryTerm.class); - label.setName(glossaryTerm.getName()); - label.setDisplayName(glossaryTerm.getDisplayName()); - label.setDescription(glossaryTerm.getDescription()); - label.setStyle(glossaryTerm.getStyle()); - } else { - throw new IllegalArgumentException("Invalid source type " + source); - } - return Pair.of(r.getString("targetFQNHash"), label); - } - } - - class TagLabelWithFQNHashMapper implements RowMapper { - @Override - public TagLabelWithFQNHash map(ResultSet rs, StatementContext ctx) throws SQLException { - TagLabelMetadata metadata = null; - try { - metadata = JsonUtils.readValue(rs.getString("metadata"), TagLabelMetadata.class); - } catch (Exception e) { - // Ignore unknown fields from future schema versions — metadata is best-effort - } - TagLabelWithFQNHash tag = new TagLabelWithFQNHash(); - tag.setTargetFQNHash(rs.getString("targetFQNHash")); - tag.setSource(rs.getInt("source")); - tag.setTagFQN(rs.getString("tagFQN")); - tag.setLabelType(rs.getInt("labelType")); - tag.setState(rs.getInt("state")); - tag.setReason(rs.getString("reason")); - tag.setAppliedAt(rs.getTimestamp("appliedAt")); - tag.setAppliedBy(rs.getString("appliedBy")); - tag.setMetadata(metadata); - return tag; - } - } - - @Getter - @Setter - class TagLabelWithFQNHash { - private String targetFQNHash; - private int source; - private String tagFQN; - private int labelType; - private int state; - private String reason; - private Date appliedAt; - private String appliedBy; - private TagLabelMetadata metadata; - - public TagLabel toTagLabel() { - TagLabel tagLabel = new TagLabel(); - TagLabel.TagSource[] sources = TagLabel.TagSource.values(); - tagLabel.setSource( - this.source >= 0 && this.source < sources.length - ? sources[this.source] - : TagLabel.TagSource.CLASSIFICATION); - tagLabel.setTagFQN(this.tagFQN); - TagLabel.LabelType[] labelTypes = TagLabel.LabelType.values(); - tagLabel.setLabelType( - this.labelType >= 0 && this.labelType < labelTypes.length - ? labelTypes[this.labelType] - : TagLabel.LabelType.MANUAL); - TagLabel.State[] states = TagLabel.State.values(); - tagLabel.setState( - this.state >= 0 && this.state < states.length - ? states[this.state] - : TagLabel.State.CONFIRMED); - tagLabel.setReason(this.reason); - tagLabel.setAppliedAt(this.appliedAt); - tagLabel.setAppliedBy(this.appliedBy); - tagLabel.setMetadata(this.metadata); - return tagLabel; - } - } - - @Getter - @Setter - @Deprecated(since = "Release 1.1") - class TagLabelMigration { - private int source; - private String tagFQN; - private String targetFQN; - private int labelType; - private int state; - private String tagFQNHash; - private String targetFQNHash; - } - - @Deprecated(since = "Release 1.1") - class TagLabelMapperMigration implements RowMapper { - @Override - public TagLabelMigration map(ResultSet r, StatementContext ctx) throws SQLException { - TagLabelMigration tagLabel = new TagLabelMigration(); - - tagLabel.setSource(r.getInt("source")); - tagLabel.setLabelType(r.getInt("labelType")); - tagLabel.setState(r.getInt("state")); - tagLabel.setTagFQN(r.getString("tagFQN")); - // TODO : Ugly , but this is present is lower version and removed on higher version - try { - // This field is removed in latest - tagLabel.setTargetFQN(r.getString("targetFQN")); - } catch (Exception ex) { - // Nothing to do - } - try { - tagLabel.setTagFQNHash(r.getString("tagFQNHash")); - } catch (Exception ex) { - // Nothing to do - } - try { - tagLabel.setTargetFQNHash(r.getString("targetFQNHash")); - } catch (Exception ex) { - // Nothing to do - } - return tagLabel; - } - } - - record TagUsageBatchRow( - int source, - String tagFQN, - String tagFQNHash, - String targetFQNHash, - int labelType, - int state, - String reason, - String appliedBy, - String metadata) {} - - record TagUsageDeleteRow(int source, String tagFQNHash, String targetFQNHash) {} - - private static String buildTagUsageKey(int source, String tagFQNHash, String targetFQNHash) { - return source + "|" + tagFQNHash + "|" + targetFQNHash; - } - - private static String buildTagUsageKey(TagUsageBatchRow row) { - return buildTagUsageKey(row.source(), row.tagFQNHash(), row.targetFQNHash()); - } - - private static String buildTagUsageKey(TagUsageDeleteRow row) { - return buildTagUsageKey(row.source(), row.tagFQNHash(), row.targetFQNHash()); - } - - Logger TAG_USAGE_LOG = LoggerFactory.getLogger(TagUsageDAO.class); - int TAG_USAGE_MAX_ATTEMPTS = 2; - AtomicLong TAG_USAGE_DEADLOCK_RETRY_COUNT = new AtomicLong(0); - - private static boolean isTransientDeadlock(Throwable throwable) { - for (Throwable current = throwable; current != null; current = current.getCause()) { - if (current instanceof SQLException sqlException) { - int errorCode = sqlException.getErrorCode(); - String sqlState = sqlException.getSQLState(); - if (errorCode == 1213 - || errorCode == 1205 - || "40001".equals(sqlState) - || "40P01".equals(sqlState)) { - return true; - } - } - } - return false; - } - - default void executeWithDeadlockRetry(Runnable operation) { - for (int attempt = 1; attempt <= TAG_USAGE_MAX_ATTEMPTS; attempt++) { - try { - operation.run(); - return; - } catch (RuntimeException ex) { - if (!isTransientDeadlock(ex) || attempt == TAG_USAGE_MAX_ATTEMPTS) { - throw ex; - } - try { - Thread.sleep(20L + (long) (Math.random() * 80)); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw ex; - } - long retryCount = TAG_USAGE_DEADLOCK_RETRY_COUNT.incrementAndGet(); - TAG_USAGE_LOG.debug( - "Retrying tag_usage batch after transient deadlock (attempt {}/{}), total retries={}", - attempt + 1, - TAG_USAGE_MAX_ATTEMPTS, - retryCount); - } - } - } - - /** - * Apply multiple tags in batch to improve performance - */ - default void applyTagsBatch(List tagLabels, String targetFQN) { - if (tagLabels == null || tagLabels.isEmpty()) { - return; - } - - String targetFQNHash = FullyQualifiedName.buildHash(targetFQN); - List rows = new ArrayList<>(tagLabels.size()); - - for (TagLabel tagLabel : tagLabels) { - rows.add( - new TagUsageBatchRow( - tagLabel.getSource().ordinal(), - tagLabel.getTagFQN(), - FullyQualifiedName.buildHash(tagLabel.getTagFQN()), - targetFQNHash, - tagLabel.getLabelType().ordinal(), - tagLabel.getState().ordinal(), - tagLabel.getReason(), - tagLabel.getAppliedBy(), - JsonUtils.pojoToJson(tagLabel.getMetadata()))); - } - - // De-duplicate duplicate tag applications within the same batch. - LinkedHashMap uniqueRows = new LinkedHashMap<>(rows.size()); - for (TagUsageBatchRow row : rows) { - uniqueRows.put(buildTagUsageKey(row), row); - } - rows = new ArrayList<>(uniqueRows.values()); - - // Deterministic lock acquisition order reduces deadlocks under concurrent upserts. - rows.sort( - java.util.Comparator.comparing(TagUsageBatchRow::targetFQNHash) - .thenComparing(TagUsageBatchRow::tagFQNHash) - .thenComparingInt(TagUsageBatchRow::source)); - - List sources = new ArrayList<>(rows.size()); - List tagFQNs = new ArrayList<>(rows.size()); - List tagFQNHashes = new ArrayList<>(rows.size()); - List targetFQNHashes = new ArrayList<>(rows.size()); - List labelTypes = new ArrayList<>(rows.size()); - List states = new ArrayList<>(rows.size()); - List reasons = new ArrayList<>(rows.size()); - List appliedBys = new ArrayList<>(rows.size()); - List metadataList = new ArrayList<>(rows.size()); - for (TagUsageBatchRow row : rows) { - sources.add(row.source()); - tagFQNs.add(row.tagFQN()); - tagFQNHashes.add(row.tagFQNHash()); - targetFQNHashes.add(row.targetFQNHash()); - labelTypes.add(row.labelType()); - states.add(row.state()); - reasons.add(row.reason()); - appliedBys.add(row.appliedBy()); - metadataList.add(row.metadata()); - } - - executeWithDeadlockRetry( - () -> - applyTagsBatchInternal( - sources, - tagFQNs, - tagFQNHashes, - targetFQNHashes, - labelTypes, - states, - reasons, - appliedBys, - metadataList)); - } - - /** - * Apply multiple tags in batch to multiple targets. Each entry in the map represents - * a target FQN and its associated tags. - */ - default void applyTagsBatchMultiTarget(Map> tagsByTarget) { - if (tagsByTarget == null || tagsByTarget.isEmpty()) { - return; - } - - List rows = new ArrayList<>(); - - for (Map.Entry> entry : tagsByTarget.entrySet()) { - String targetFQN = entry.getKey(); - List tagLabels = entry.getValue(); - if (tagLabels == null || tagLabels.isEmpty()) { - continue; - } - - String targetFQNHash = FullyQualifiedName.buildHash(targetFQN); - for (TagLabel tagLabel : tagLabels) { - if (tagLabel.getLabelType().equals(TagLabel.LabelType.DERIVED)) { - continue; - } - rows.add( - new TagUsageBatchRow( - tagLabel.getSource().ordinal(), - tagLabel.getTagFQN(), - FullyQualifiedName.buildHash(tagLabel.getTagFQN()), - targetFQNHash, - tagLabel.getLabelType().ordinal(), - tagLabel.getState().ordinal(), - tagLabel.getReason(), - tagLabel.getAppliedBy(), - JsonUtils.pojoToJson(tagLabel.getMetadata()))); - } - } - - if (!rows.isEmpty()) { - // De-duplicate duplicate tag applications within the same batch. - LinkedHashMap uniqueRows = new LinkedHashMap<>(rows.size()); - for (TagUsageBatchRow row : rows) { - uniqueRows.put(buildTagUsageKey(row), row); - } - rows = new ArrayList<>(uniqueRows.values()); - - // Deterministic lock acquisition order reduces deadlocks under concurrent upserts. - rows.sort( - java.util.Comparator.comparing(TagUsageBatchRow::targetFQNHash) - .thenComparing(TagUsageBatchRow::tagFQNHash) - .thenComparingInt(TagUsageBatchRow::source)); - - List sources = new ArrayList<>(rows.size()); - List tagFQNs = new ArrayList<>(rows.size()); - List tagFQNHashes = new ArrayList<>(rows.size()); - List targetFQNHashes = new ArrayList<>(rows.size()); - List labelTypes = new ArrayList<>(rows.size()); - List states = new ArrayList<>(rows.size()); - List reasons = new ArrayList<>(rows.size()); - List appliedBys = new ArrayList<>(rows.size()); - List metadataList = new ArrayList<>(rows.size()); - for (TagUsageBatchRow row : rows) { - sources.add(row.source()); - tagFQNs.add(row.tagFQN()); - tagFQNHashes.add(row.tagFQNHash()); - targetFQNHashes.add(row.targetFQNHash()); - labelTypes.add(row.labelType()); - states.add(row.state()); - reasons.add(row.reason()); - appliedBys.add(row.appliedBy()); - metadataList.add(row.metadata()); - } - - executeWithDeadlockRetry( - () -> - applyTagsBatchInternal( - sources, - tagFQNs, - tagFQNHashes, - targetFQNHashes, - labelTypes, - states, - reasons, - appliedBys, - metadataList)); - } - } - - @Transaction - @ConnectionAwareSqlBatch( - value = - "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedBy, metadata) " - + "VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :reason, :appliedBy, :metadata) " - + "ON DUPLICATE KEY UPDATE labelType = VALUES(labelType), state = VALUES(state), reason = VALUES(reason), metadata = VALUES(metadata)", - connectionType = MYSQL) - @ConnectionAwareSqlBatch( - value = - "INSERT INTO tag_usage (source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedBy, metadata) " - + "VALUES (:source, :tagFQN, :tagFQNHash, :targetFQNHash, :labelType, :state, :reason, :appliedBy, :metadata :: jsonb) " - + "ON CONFLICT (source, tagFQNHash, targetFQNHash) DO UPDATE SET labelType = EXCLUDED.labelType, " - + "state = EXCLUDED.state, reason = EXCLUDED.reason, metadata = EXCLUDED.metadata", - connectionType = POSTGRES) - void applyTagsBatchInternal( - @Bind("source") List sources, - @Bind("tagFQN") List tagFQNs, - @Bind("tagFQNHash") List tagFQNHashes, - @Bind("targetFQNHash") List targetFQNHashes, - @Bind("labelType") List labelTypes, - @Bind("state") List states, - @Bind("reason") List reasons, - @Bind("appliedBy") List appliedBys, - @Bind("metadata") List metadataList); - - /** - * Delete multiple tags in batch to improve performance - */ - default void deleteTagsBatch(List tagLabels, String targetFQN) { - if (tagLabels == null || tagLabels.isEmpty()) { - return; - } - - String targetFQNHash = FullyQualifiedName.buildHash(targetFQN); - List rows = new ArrayList<>(tagLabels.size()); - - for (TagLabel tagLabel : tagLabels) { - rows.add( - new TagUsageDeleteRow( - tagLabel.getSource().ordinal(), - FullyQualifiedName.buildHash(tagLabel.getTagFQN()), - targetFQNHash)); - } - - LinkedHashMap uniqueRows = new LinkedHashMap<>(rows.size()); - for (TagUsageDeleteRow row : rows) { - uniqueRows.put(buildTagUsageKey(row), row); - } - rows = new ArrayList<>(uniqueRows.values()); - - rows.sort( - java.util.Comparator.comparing(TagUsageDeleteRow::targetFQNHash) - .thenComparing(TagUsageDeleteRow::tagFQNHash) - .thenComparingInt(TagUsageDeleteRow::source)); - - List sources = new ArrayList<>(rows.size()); - List tagFQNHashes = new ArrayList<>(rows.size()); - List targetFQNHashes = new ArrayList<>(rows.size()); - for (TagUsageDeleteRow row : rows) { - sources.add(row.source()); - tagFQNHashes.add(row.tagFQNHash()); - targetFQNHashes.add(row.targetFQNHash()); - } - - executeWithDeadlockRetry( - () -> deleteTagsBatchInternal(sources, tagFQNHashes, targetFQNHashes)); - } - - @Transaction - @ConnectionAwareSqlBatch( - value = - "DELETE FROM tag_usage WHERE source = :source AND tagFQNHash = :tagFQNHash AND targetFQNHash = :targetFQNHash", - connectionType = MYSQL) - @ConnectionAwareSqlBatch( - value = - "DELETE FROM tag_usage WHERE source = :source AND tagFQNHash = :tagFQNHash AND targetFQNHash = :targetFQNHash", - connectionType = POSTGRES) - void deleteTagsBatchInternal( - @Bind("source") List sources, - @Bind("tagFQNHash") List tagFQNHashes, - @Bind("targetFQNHash") List targetFQNHashes); - - @SqlQuery("SELECT COUNT(*) FROM tag_usage") - long getTotalTagUsageCount(); - - @SqlQuery( - "SELECT source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedAt, appliedBy, metadata FROM tag_usage ORDER BY source, tagFQNHash LIMIT :limit OFFSET :offset") - @RegisterRowMapper(TagUsageObjectMapper.class) - List getAllTagUsagesPaginated( - @Bind("offset") long offset, @Bind("limit") int limit); - - @SqlUpdate( - "DELETE FROM tag_usage WHERE source = :source AND tagFQNHash = :tagFQNHash AND targetFQNHash = :targetFQNHash") - int deleteTagUsage( - @Bind("source") int source, - @Bind("tagFQNHash") String tagFQNHash, - @Bind("targetFQNHash") String targetFQNHash); - } - - @Getter - @Builder - class TagUsageObject { - private int source; - private String tagFQN; - private String tagFQNHash; - private String targetFQNHash; - private int labelType; - private int state; - private String reason; - private Date appliedAt; - private String appliedBy; - private TagLabelMetadata metadata; - } - - class TagUsageObjectMapper implements RowMapper { - @Override - public TagUsageObject map(ResultSet r, StatementContext ctx) throws SQLException { - return TagUsageObject.builder() - .source(r.getInt("source")) - .tagFQN(r.getString("tagFQN")) - .tagFQNHash(r.getString("tagFQNHash")) - .targetFQNHash(r.getString("targetFQNHash")) - .labelType(r.getInt("labelType")) - .state(r.getInt("state")) - .reason(r.getString("reason")) - .appliedAt(r.getTimestamp("appliedAt")) - .appliedBy(r.getString("appliedBy")) - .metadata(JsonUtils.readValue(r.getString("metadata"), TagLabelMetadata.class)) - .build(); - } - } - - interface RoleDAO extends EntityDAO { - @Override - default String getTableName() { - return "role_entity"; - } - - @Override - default Class getEntityClass() { - return Role.class; - } - } - - interface PersonaDAO extends EntityDAO { - @Override - default String getTableName() { - return "persona_entity"; - } - - @Override - default Class getEntityClass() { - return Persona.class; - } - - @Override - default boolean supportsSoftDelete() { - return false; - } - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM persona_entity WHERE JSON_EXTRACT(json, '$.default') = true LIMIT 1", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT json FROM persona_entity WHERE json->>'default' = 'true' LIMIT 1", - connectionType = POSTGRES) - String findDefaultPersona(); - - @ConnectionAwareSqlQuery( - value = - "SELECT id FROM persona_entity WHERE JSON_EXTRACT(json, '$.default') = true AND id != :excludeId", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT id FROM persona_entity WHERE json->>'default' = 'true' AND id != :excludeId", - connectionType = POSTGRES) - List findOtherDefaultPersonaIds(@Bind("excludeId") String excludeId); - - @ConnectionAwareSqlQuery( - value = - "SELECT id, JSON_UNQUOTE(JSON_EXTRACT(json, '$.fullyQualifiedName')) AS fqn " - + "FROM persona_entity " - + "WHERE JSON_EXTRACT(json, '$.default') = true AND id != :excludeId", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT id, json->>'fullyQualifiedName' AS fqn FROM persona_entity " - + "WHERE json->>'default' = 'true' AND id != :excludeId", - connectionType = POSTGRES) - @RegisterRowMapper(EntityDAO.EntityIdFqnPairMapper.class) - List findOtherDefaultPersonaIdsWithFqn( - @Bind("excludeId") String excludeId); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE persona_entity SET json = JSON_SET(json, '$.default', false) WHERE JSON_EXTRACT(json, '$.default') = true AND id != :excludeId", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE persona_entity SET json = jsonb_set(json, '{default}', 'false') WHERE json->>'default' = 'true' AND id != :excludeId", - connectionType = POSTGRES) - void unsetOtherDefaultPersonas(@Bind("excludeId") String excludeId); - } - - interface TeamDAO extends EntityDAO { - @Override - default String getTableName() { - return "team_entity"; - } - - @Override - default Class getEntityClass() { - return Team.class; - } - - @Override - default int listCount(ListFilter filter) { - String parentTeam = filter.getQueryParam("parentTeam"); - String isJoinable = filter.getQueryParam("isJoinable"); - String condition = filter.getCondition(); - if (parentTeam != null) { - // validate parent team - Team team = findEntityByName(parentTeam, Include.ALL); - if (ORGANIZATION_NAME.equals(team.getName())) { - // All the teams without parents should come under "organization" team - condition = - String.format( - "%s AND id NOT IN ( (SELECT '%s') UNION (SELECT toId FROM entity_relationship WHERE fromId!='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d) )", - condition, team.getId(), team.getId(), Relationship.PARENT_OF.ordinal()); - } else { - condition = - String.format( - "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d)", - condition, team.getId(), Relationship.PARENT_OF.ordinal()); - } - } - String mySqlCondition = condition; - String postgresCondition = condition; - if (isJoinable != null) { - mySqlCondition = - String.format( - "%s AND JSON_EXTRACT(json, '$.isJoinable') = :isJoinable ", mySqlCondition); - postgresCondition = - String.format( - "%s AND ((json#>'{isJoinable}')::boolean) = :isJoinable ", postgresCondition); - } - - return listCount( - getTableName(), - getNameHashColumn(), - filter.getQueryParams(), - mySqlCondition, - postgresCondition); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String parentTeam = filter.getQueryParam("parentTeam"); - String isJoinable = filter.getQueryParam("isJoinable"); - String condition = filter.getCondition(); - if (parentTeam != null) { - // validate parent team - Team team = findEntityByName(parentTeam, Include.ALL); - if (ORGANIZATION_NAME.equals(team.getName())) { - // All the parentless teams should come under "organization" team - condition = - String.format( - "%s AND id NOT IN ( (SELECT '%s') UNION (SELECT toId FROM entity_relationship WHERE fromId!='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d) )", - condition, team.getId(), team.getId(), Relationship.PARENT_OF.ordinal()); - } else { - condition = - String.format( - "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d)", - condition, team.getId(), Relationship.PARENT_OF.ordinal()); - } - } - String mySqlCondition = condition; - String postgresCondition = condition; - if (isJoinable != null) { - mySqlCondition = - String.format( - "%s AND JSON_EXTRACT(json, '$.isJoinable') = :isJoinable ", mySqlCondition); - postgresCondition = - String.format( - "%s AND ((json#>'{isJoinable}')::boolean) = :isJoinable ", postgresCondition); - } - - // Quoted name is stored in fullyQualifiedName column and not in the name column - beforeName = - Optional.ofNullable(beforeName).map(FullyQualifiedName::unquoteName).orElse(null); - return listBefore( - getTableName(), - filter.getQueryParams(), - mySqlCondition, - postgresCondition, - limit, - beforeName, - beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String parentTeam = filter.getQueryParam("parentTeam"); - String isJoinable = filter.getQueryParam("isJoinable"); - String condition = filter.getCondition(); - if (parentTeam != null) { - // validate parent team - Team team = findEntityByName(parentTeam, Include.ALL); - if (ORGANIZATION_NAME.equals(team.getName())) { - // All the parentless teams should come under "organization" team - condition = - String.format( - "%s AND id NOT IN ( (SELECT '%s') UNION (SELECT toId FROM entity_relationship WHERE fromId!='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d) )", - condition, team.getId(), team.getId(), Relationship.PARENT_OF.ordinal()); - } else { - condition = - String.format( - "%s AND id IN (SELECT toId FROM entity_relationship WHERE fromId='%s' AND fromEntity='team' AND toEntity='team' AND relation=%d)", - condition, team.getId(), Relationship.PARENT_OF.ordinal()); - } - } - String mySqlCondition = condition; - String postgresCondition = condition; - if (isJoinable != null) { - mySqlCondition = - String.format( - "%s AND JSON_EXTRACT(json, '$.isJoinable') = %s ", mySqlCondition, isJoinable); - postgresCondition = - String.format( - "%s AND ((json#>'{isJoinable}')::boolean) = %s ", postgresCondition, isJoinable); - } - - // Quoted name is stored in fullyQualifiedName column and not in the name column - afterName = Optional.ofNullable(afterName).map(FullyQualifiedName::unquoteName).orElse(null); - return listAfter( - getTableName(), - filter.getQueryParams(), - mySqlCondition, - postgresCondition, - limit, - afterName, - afterId); - } - - default List listTeamsUnderOrganization(UUID teamId) { - return listTeamsUnderOrganization(teamId, Relationship.PARENT_OF.ordinal()); - } - - @SqlQuery( - "SELECT te.id " - + "FROM team_entity te " - + "WHERE te.id NOT IN ((SELECT :teamId) UNION " - + "(SELECT toId FROM entity_relationship " - + "WHERE fromId != :teamId AND fromEntity = 'team' AND relation = :relation AND toEntity = 'team'))") - List listTeamsUnderOrganization( - @BindUUID("teamId") UUID teamId, @Bind("relation") int relation); - } - - interface TopicDAO extends EntityDAO { - @Override - default String getTableName() { - return "topic_entity"; - } - - @Override - default Class getEntityClass() { - return Topic.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - @RegisterRowMapper(UsageDetailsMapper.class) - interface UsageDAO { - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_usage (usageDate, id, entityType, count1, count7, count30) " - + "SELECT :date, :id, :entityType, :count1, " - + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= :date - " - + "INTERVAL 6 DAY)), " - + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= :date - " - + "INTERVAL 29 DAY))" - + "ON DUPLICATE KEY UPDATE count7 = count7 - count1 + :count1, count30 = count30 - count1 + :count1, count1 = :count1", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_usage (usageDate, id, entityType, count1, count7, count30) " - + "SELECT (:date :: date), :id, :entityType, :count1, " - + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= (:date :: date) - INTERVAL '6 days')), " - + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= (:date :: date) - INTERVAL '29 days'))" - + "ON CONFLICT (usageDate, id) DO UPDATE SET count7 = entity_usage.count7 - entity_usage.count1 + :count1," - + "count30 = entity_usage.count30 - entity_usage.count1 + :count1, count1 = :count1", - connectionType = POSTGRES) - void insertOrReplaceCount( - @Bind("date") String date, - @BindUUID("id") UUID id, - @Bind("entityType") String entityType, - @Bind("count1") int count1); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_usage (usageDate, id, entityType, count1, count7, count30) " - + "SELECT :date, :id, :entityType, :count1, " - + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= :date - " - + "INTERVAL 6 DAY)), " - + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= :date - " - + "INTERVAL 29 DAY)) " - + "ON DUPLICATE KEY UPDATE count1 = count1 + :count1, count7 = count7 + :count1, count30 = count30 + :count1", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO entity_usage (usageDate, id, entityType, count1, count7, count30) " - + "SELECT (:date :: date), :id, :entityType, :count1, " - + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= (:date :: date) - INTERVAL '6 days')), " - + "(:count1 + (SELECT COALESCE(SUM(count1), 0) FROM entity_usage WHERE id = :id AND usageDate >= (:date :: date) - INTERVAL '29 days')) " - + "ON CONFLICT (usageDate, id) DO UPDATE SET count1 = entity_usage.count1 + :count1, count7 = entity_usage.count7 + :count1, count30 = entity_usage.count30 + :count1", - connectionType = POSTGRES) - void insertOrUpdateCount( - @Bind("date") String date, - @BindUUID("id") UUID id, - @Bind("entityType") String entityType, - @Bind("count1") int count1); - - @ConnectionAwareSqlQuery( - value = - "SELECT id, usageDate, entityType, count1, count7, count30, " - + "percentile1, percentile7, percentile30 FROM entity_usage " - + "WHERE id = :id AND usageDate >= :date - INTERVAL :days DAY AND usageDate <= :date ORDER BY usageDate DESC", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT id, usageDate, entityType, count1, count7, count30, " - + "percentile1, percentile7, percentile30 FROM entity_usage " - + "WHERE id = :id AND usageDate >= (:date :: date) - make_interval(days => :days) AND usageDate <= (:date :: date) ORDER BY usageDate DESC", - connectionType = POSTGRES) - List getUsageById( - @BindUUID("id") UUID id, @Bind("date") String date, @Bind("days") int days); - - /** Get latest usage record */ - @SqlQuery( - "SELECT id, usageDate, entityType, count1, count7, count30, " - + "percentile1, percentile7, percentile30 FROM entity_usage " - + "WHERE usageDate IN (SELECT MAX(usageDate) FROM entity_usage WHERE id = :id) AND id = :id") - UsageDetails getLatestUsage(@Bind("id") String id); - - /** Get latest usage records for multiple entities in one query */ - @RegisterRowMapper(UsageDetailsWithIdMapper.class) - @SqlQuery( - "SELECT u1.id, u1.usageDate, u1.entityType, u1.count1, u1.count7, u1.count30, " - + "u1.percentile1, u1.percentile7, u1.percentile30 FROM entity_usage u1 " - + "INNER JOIN (SELECT id, MAX(usageDate) as maxDate FROM entity_usage WHERE id IN () GROUP BY id) u2 " - + "ON u1.id = u2.id AND u1.usageDate = u2.maxDate") - List getLatestUsageBatch(@BindList("ids") List ids); - - @SqlUpdate("DELETE FROM entity_usage WHERE id = :id") - void delete(@BindUUID("id") UUID id); - - /** - * TODO: Not sure I get what the next comment means, but tests now use mysql 8 so maybe tests can be improved here - * Note not using in following percentile computation PERCENT_RANK function as unit tests use mysql5.7, and it does - * not have window function - */ - @ConnectionAwareSqlUpdate( - value = - "UPDATE entity_usage u JOIN ( " - + "SELECT u1.id, " - + "(SELECT COUNT(*) FROM entity_usage as u2 WHERE u2.count1 < u1.count1 AND u2.entityType = :entityType " - + "AND u2.usageDate = :date) as p1, " - + "(SELECT COUNT(*) FROM entity_usage as u3 WHERE u3.count7 < u1.count7 AND u3.entityType = :entityType " - + "AND u3.usageDate = :date) as p7, " - + "(SELECT COUNT(*) FROM entity_usage as u4 WHERE u4.count30 < u1.count30 AND u4.entityType = :entityType " - + "AND u4.usageDate = :date) as p30, " - + "(SELECT COUNT(*) FROM entity_usage WHERE entityType = :entityType AND usageDate = :date) as total " - + "FROM entity_usage u1 WHERE u1.entityType = :entityType AND u1.usageDate = :date" - + ") vals ON u.id = vals.id AND usageDate = :date " - + "SET u.percentile1 = ROUND(100 * p1/total, 2), u.percentile7 = ROUND(p7 * 100/total, 2), u.percentile30 =" - + " ROUND(p30*100/total, 2)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE entity_usage u " - + "SET percentile1 = ROUND(100 * p1 / total, 2), percentile7 = ROUND(p7 * 100 / total, 2), percentile30 = ROUND(p30 * 100 / total, 2) " - + "FROM (" - + " SELECT u1.id, " - + " (SELECT COUNT(*) FROM entity_usage as u2 WHERE u2.count1 < u1.count1 AND u2.entityType = :entityType AND u2.usageDate = (:date :: date)) as p1, " - + " (SELECT COUNT(*) FROM entity_usage as u3 WHERE u3.count7 < u1.count7 AND u3.entityType = :entityType AND u3.usageDate = (:date :: date)) as p7, " - + " (SELECT COUNT(*) FROM entity_usage as u4 WHERE u4.count30 < u1.count30 AND u4.entityType = :entityType AND u4.usageDate = (:date :: date)) as p30, " - + " (SELECT COUNT(*) FROM entity_usage WHERE entityType = :entityType AND usageDate = (:date :: date)" - + " ) as total FROM entity_usage u1 " - + " WHERE u1.entityType = :entityType AND u1.usageDate = (:date :: date)" - + ") vals " - + "WHERE u.id = vals.id AND usageDate = (:date :: date);", - connectionType = POSTGRES) - void computePercentile(@Bind("entityType") String entityType, @Bind("date") String date); - - class UsageDetailsMapper implements RowMapper { - @Override - public UsageDetails map(ResultSet r, StatementContext ctx) throws SQLException { - UsageStats dailyStats = - new UsageStats() - .withCount(r.getInt("count1")) - .withPercentileRank(r.getDouble("percentile1")); - UsageStats weeklyStats = - new UsageStats() - .withCount(r.getInt("count7")) - .withPercentileRank(r.getDouble("percentile7")); - UsageStats monthlyStats = - new UsageStats() - .withCount(r.getInt("count30")) - .withPercentileRank(r.getDouble("percentile30")); - java.sql.Date usageDate = r.getDate("usageDate"); - return new UsageDetails() - .withDate(usageDate != null ? usageDate.toString() : null) - .withDailyStats(dailyStats) - .withWeeklyStats(weeklyStats) - .withMonthlyStats(monthlyStats); - } - } - - /** Usage details with entity ID for batch operations */ - class UsageDetailsWithId { - private final String entityId; - private final UsageDetails usageDetails; - - public UsageDetailsWithId(String entityId, UsageDetails usageDetails) { - this.entityId = entityId; - this.usageDetails = usageDetails; - } - - public String getEntityId() { - return entityId; - } - - public UsageDetails getUsageDetails() { - return usageDetails; - } - } - - class UsageDetailsWithIdMapper implements RowMapper { - @Override - public UsageDetailsWithId map(ResultSet r, StatementContext ctx) throws SQLException { - String entityId = r.getString("id"); - UsageStats dailyStats = - new UsageStats() - .withCount(r.getInt("count1")) - .withPercentileRank(r.getDouble("percentile1")); - UsageStats weeklyStats = - new UsageStats() - .withCount(r.getInt("count7")) - .withPercentileRank(r.getDouble("percentile7")); - UsageStats monthlyStats = - new UsageStats() - .withCount(r.getInt("count30")) - .withPercentileRank(r.getDouble("percentile30")); - java.sql.Date usageDate = r.getDate("usageDate"); - UsageDetails usageDetails = - new UsageDetails() - .withDate(usageDate != null ? usageDate.toString() : null) - .withDailyStats(dailyStats) - .withWeeklyStats(weeklyStats) - .withMonthlyStats(monthlyStats); - return new UsageDetailsWithId(entityId, usageDetails); - } - } - } - - interface UserDAO extends EntityDAO { - @Override - default String getTableName() { - return "user_entity"; - } - - @Override - default Class getEntityClass() { - return User.class; - } - - @Override - default int listCount(ListFilter filter) { - String team = EntityInterfaceUtil.quoteName(filter.getQueryParam("team")); - String isBotStr = filter.getQueryParam("isBot"); - String isAdminStr = filter.getQueryParam("isAdmin"); - String lastLoginTimeGreaterThan = filter.getQueryParam("lastLoginTimeGreaterThan"); - String lastActivityTimeGreaterThan = filter.getQueryParam("lastActivityTimeGreaterThan"); - String mySqlCondition = filter.getCondition("ue"); - String postgresCondition = filter.getCondition("ue"); - if (isAdminStr != null) { - boolean isAdmin = Boolean.parseBoolean(isAdminStr); - if (isAdmin) { - mySqlCondition = - String.format("%s AND JSON_EXTRACT(ue.json, '$.isAdmin') = TRUE ", mySqlCondition); - postgresCondition = - String.format("%s AND ((ue.json#>'{isAdmin}')::boolean) = TRUE ", postgresCondition); - } else { - mySqlCondition = - String.format( - "%s AND (JSON_EXTRACT(ue.json, '$.isAdmin') IS NULL OR JSON_EXTRACT(ue.json, '$.isAdmin') = FALSE ) ", - mySqlCondition); - postgresCondition = - String.format( - "%s AND (ue.json#>'{isAdmin}' IS NULL OR ((ue.json#>'{isAdmin}')::boolean) = FALSE ) ", - postgresCondition); - } - } - if (isBotStr != null) { - boolean isBot = Boolean.parseBoolean(isBotStr); - if (isBot) { - mySqlCondition = - String.format("%s AND JSON_EXTRACT(ue.json, '$.isBot') = TRUE ", mySqlCondition); - postgresCondition = - String.format("%s AND ((ue.json#>'{isBot}')::boolean) = TRUE ", postgresCondition); - } else { - mySqlCondition = - String.format( - "%s AND (JSON_EXTRACT(ue.json, '$.isBot') IS NULL OR JSON_EXTRACT(ue.json, '$.isBot') = FALSE ) ", - mySqlCondition); - postgresCondition = - String.format( - "%s AND (ue.json#>'{isBot}' IS NULL OR ((ue.json#>'{isBot}')::boolean) = FALSE) ", - postgresCondition); - } - } - if (lastLoginTimeGreaterThan != null) { - mySqlCondition = - String.format( - "%s AND ue.lastLoginTime > %s ", mySqlCondition, lastLoginTimeGreaterThan); - postgresCondition = - String.format( - "%s AND ue.lastLoginTime > %s ", postgresCondition, lastLoginTimeGreaterThan); - } - if (lastActivityTimeGreaterThan != null) { - mySqlCondition = - String.format( - "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", - mySqlCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); - postgresCondition = - String.format( - "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", - postgresCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); - } - if (team == null - && isAdminStr == null - && isBotStr == null - && lastLoginTimeGreaterThan == null - && lastActivityTimeGreaterThan == null) { - return EntityDAO.super.listCount(filter); - } - return listCount( - getTableName(), mySqlCondition, postgresCondition, team, Relationship.HAS.ordinal()); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String team = EntityInterfaceUtil.quoteName(filter.getQueryParam("team")); - String isBotStr = filter.getQueryParam("isBot"); - String isAdminStr = filter.getQueryParam("isAdmin"); - String lastLoginTimeGreaterThan = filter.getQueryParam("lastLoginTimeGreaterThan"); - String lastActivityTimeGreaterThan = filter.getQueryParam("lastActivityTimeGreaterThan"); - String mySqlCondition = filter.getCondition("ue"); - String postgresCondition = filter.getCondition("ue"); - if (isAdminStr != null) { - boolean isAdmin = Boolean.parseBoolean(isAdminStr); - if (isAdmin) { - mySqlCondition = - String.format("%s AND JSON_EXTRACT(ue.json, '$.isAdmin') = TRUE ", mySqlCondition); - postgresCondition = - String.format("%s AND ((ue.json#>'{isAdmin}')::boolean) = TRUE ", postgresCondition); - } else { - mySqlCondition = - String.format( - "%s AND (JSON_EXTRACT(ue.json, '$.isAdmin') IS NULL OR JSON_EXTRACT(ue.json, '$.isAdmin') = FALSE ) ", - mySqlCondition); - postgresCondition = - String.format( - "%s AND (ue.json#>'{isAdmin}' IS NULL OR ((ue.json#>'{isAdmin}')::boolean) = FALSE ) ", - postgresCondition); - } - } - if (isBotStr != null) { - boolean isBot = Boolean.parseBoolean(isBotStr); - if (isBot) { - mySqlCondition = - String.format("%s AND JSON_EXTRACT(ue.json, '$.isBot') = TRUE ", mySqlCondition); - postgresCondition = - String.format("%s AND ((ue.json#>'{isBot}')::boolean) = TRUE ", postgresCondition); - } else { - mySqlCondition = - String.format( - "%s AND (JSON_EXTRACT(ue.json, '$.isBot') IS NULL OR JSON_EXTRACT(ue.json, '$.isBot') = FALSE ) ", - mySqlCondition); - postgresCondition = - String.format( - "%s AND (ue.json#>'{isBot}' IS NULL OR ((ue.json#>'{isBot}')::boolean) = FALSE) ", - postgresCondition); - } - } - if (lastLoginTimeGreaterThan != null) { - mySqlCondition = - String.format( - "%s AND ue.lastLoginTime > %s ", mySqlCondition, lastLoginTimeGreaterThan); - postgresCondition = - String.format( - "%s AND ue.lastLoginTime > %s ", postgresCondition, lastLoginTimeGreaterThan); - } - if (lastActivityTimeGreaterThan != null) { - mySqlCondition = - String.format( - "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", - mySqlCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); - postgresCondition = - String.format( - "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", - postgresCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); - } - if (team == null - && isAdminStr == null - && isBotStr == null - && lastLoginTimeGreaterThan == null - && lastActivityTimeGreaterThan == null) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - return listBefore( - getTableName(), - mySqlCondition, - postgresCondition, - team, - limit, - beforeName, - beforeId, - Relationship.HAS.ordinal()); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String team = EntityInterfaceUtil.quoteName(filter.getQueryParam("team")); - String isBotStr = filter.getQueryParam("isBot"); - String isAdminStr = filter.getQueryParam("isAdmin"); - String lastLoginTimeGreaterThan = filter.getQueryParam("lastLoginTimeGreaterThan"); - String lastActivityTimeGreaterThan = filter.getQueryParam("lastActivityTimeGreaterThan"); - String mySqlCondition = filter.getCondition("ue"); - String postgresCondition = filter.getCondition("ue"); - if (isAdminStr != null) { - boolean isAdmin = Boolean.parseBoolean(isAdminStr); - if (isAdmin) { - mySqlCondition = - String.format("%s AND JSON_EXTRACT(ue.json, '$.isAdmin') = TRUE ", mySqlCondition); - postgresCondition = - String.format("%s AND ((ue.json#>'{isAdmin}')::boolean) = TRUE ", postgresCondition); - } else { - mySqlCondition = - String.format( - "%s AND (JSON_EXTRACT(ue.json, '$.isAdmin') IS NULL OR JSON_EXTRACT(ue.json, '$.isAdmin') = FALSE ) ", - mySqlCondition); - postgresCondition = - String.format( - "%s AND (ue.json#>'{isAdmin}' IS NULL OR ((ue.json#>'{isAdmin}')::boolean) = FALSE ) ", - postgresCondition); - } - } - if (isBotStr != null) { - boolean isBot = Boolean.parseBoolean(isBotStr); - if (isBot) { - mySqlCondition = - String.format("%s AND JSON_EXTRACT(ue.json, '$.isBot') = TRUE ", mySqlCondition); - postgresCondition = - String.format("%s AND ((ue.json#>'{isBot}')::boolean) = TRUE ", postgresCondition); - } else { - mySqlCondition = - String.format( - "%s AND (JSON_EXTRACT(ue.json, '$.isBot') IS NULL OR JSON_EXTRACT(ue.json, '$.isBot') = FALSE ) ", - mySqlCondition); - postgresCondition = - String.format( - "%s AND (ue.json#>'{isBot}' IS NULL OR ((ue.json#>'{isBot}')::boolean) = FALSE) ", - postgresCondition); - } - } - if (lastLoginTimeGreaterThan != null) { - mySqlCondition = - String.format( - "%s AND ue.lastLoginTime > %s ", mySqlCondition, lastLoginTimeGreaterThan); - postgresCondition = - String.format( - "%s AND ue.lastLoginTime > %s ", postgresCondition, lastLoginTimeGreaterThan); - } - if (lastActivityTimeGreaterThan != null) { - mySqlCondition = - String.format( - "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", - mySqlCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); - postgresCondition = - String.format( - "%s AND ((ue.lastActivityTime IS NOT NULL AND ue.lastActivityTime > %s) OR (ue.lastLoginTime IS NOT NULL AND ue.lastLoginTime > %s)) ", - postgresCondition, lastActivityTimeGreaterThan, lastActivityTimeGreaterThan); - } - if (team == null - && isAdminStr == null - && isBotStr == null - && lastLoginTimeGreaterThan == null - && lastActivityTimeGreaterThan == null) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - return listAfter( - getTableName(), - mySqlCondition, - postgresCondition, - team, - limit, - afterName, - afterId, - Relationship.HAS.ordinal()); - } - - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM (" - + "SELECT ue.id " - + "FROM user_entity ue " - + "LEFT JOIN entity_relationship er on ue.id = er.toId " - + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " - + " " - + " AND (:team IS NULL OR te.nameHash = :team) " - + "GROUP BY ue.id) subquery", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(id) FROM (" - + "SELECT ue.id " - + "FROM user_entity ue " - + "LEFT JOIN entity_relationship er on ue.id = er.toId " - + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " - + " " - + " AND (:team IS NULL OR te.nameHash = :team) " - + "GROUP BY ue.id) subquery", - connectionType = POSTGRES) - int listCount( - @Define("table") String table, - @Define("mysqlCond") String mysqlCond, - @Define("postgresCond") String postgresCond, - @BindFQN("team") String team, - @Bind("relation") int relation); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT ue.name, ue.id, ue.json " - + "FROM user_entity ue " - + "LEFT JOIN entity_relationship er on ue.id = er.toId " - + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " - + " " - + "AND (:team IS NULL OR te.nameHash = :team) " - + "AND (ue.name < :beforeName OR (ue.name = :beforeName AND ue.id < :beforeId)) " - + "GROUP BY ue.name, ue.id, ue.json " - + "ORDER BY ue.name DESC,ue.id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT ue.name, ue.id, ue.json " - + "FROM user_entity ue " - + "LEFT JOIN entity_relationship er on ue.id = er.toId " - + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " - + " " - + "AND (:team IS NULL OR te.nameHash = :team) " - + "AND (ue.name < :beforeName OR (ue.name = :beforeName AND ue.id < :beforeId)) " - + "GROUP BY ue.name, ue.id, ue.json " - + "ORDER BY ue.name DESC,ue.id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id", - connectionType = POSTGRES) - List listBefore( - @Define("table") String table, - @Define("mysqlCond") String mysqlCond, - @Define("postgresCond") String postgresCond, - @BindFQN("team") String team, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId, - @Bind("relation") int relation); - - @ConnectionAwareSqlQuery( - value = - "SELECT ue.json " - + "FROM user_entity ue " - + "LEFT JOIN entity_relationship er on ue.id = er.toId " - + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " - + " " - + "AND (:team IS NULL OR te.nameHash = :team) " - + "AND (ue.name > :afterName OR (ue.name = :afterName AND ue.id > :afterId)) " - + "GROUP BY ue.name, ue.id, ue.json " - + "ORDER BY ue.name,ue.id " - + "LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT ue.json " - + "FROM user_entity ue " - + "LEFT JOIN entity_relationship er on ue.id = er.toId " - + "LEFT JOIN team_entity te on te.id = er.fromId and er.relation = :relation " - + " " - + "AND (:team IS NULL OR te.nameHash = :team) " - + "AND (ue.name > :afterName OR (ue.name = :afterName AND ue.id > :afterId)) " - + "GROUP BY ue.name,ue.id, ue.json " - + "ORDER BY ue.name,ue.id " - + "LIMIT :limit", - connectionType = POSTGRES) - List listAfter( - @Define("table") String table, - @Define("mysqlCond") String mysqlCond, - @Define("postgresCond") String postgresCond, - @BindFQN("team") String team, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId, - @Bind("relation") int relation); - - @SqlQuery("SELECT COUNT(*) FROM user_entity WHERE LOWER(email) = LOWER(:email)") - int checkEmailExists(@Bind("email") String email); - - @SqlQuery("SELECT COUNT(*) FROM user_entity WHERE LOWER(name) = LOWER(:name)") - int checkUserNameExists(@Bind("name") String name); - - @SqlQuery( - "SELECT json FROM user_entity WHERE LOWER(name) = LOWER(:name) AND LOWER(email) = LOWER(:email)") - String findUserByNameAndEmail(@Bind("name") String name, @Bind("email") String email); - - @SqlQuery("SELECT json FROM user_entity WHERE LOWER(email) = LOWER(:email)") - String findUserByEmail(@Bind("email") String email); - - @Override - default User findEntityByName(String fqn, Include include) { - return EntityDAO.super.findEntityByName(fqn.toLowerCase(), include); - } - - @ConnectionAwareSqlUpdate( - value = - "UPDATE user_entity SET json = JSON_SET(json, '$.lastActivityTime', :lastActivityTime) WHERE nameHash = :nameHash AND deleted = false", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE user_entity SET json = jsonb_set(json, '{lastActivityTime}', to_jsonb(:lastActivityTime::bigint)) WHERE nameHash = :nameHash AND deleted = false", - connectionType = POSTGRES) - void updateLastActivityTime( - @BindFQN("nameHash") String nameHash, @Bind("lastActivityTime") long lastActivityTime); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE user_entity SET json = JSON_SET(json, '$.lastActivityTime', " - + "CASE nameHash " - + " " - + "END) " - + "WHERE nameHash IN () AND deleted = false", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE user_entity SET json = jsonb_set(json, '{lastActivityTime}', " - + "CASE nameHash " - + " " - + "END::text::jsonb) " - + "WHERE nameHash IN () AND deleted = false", - connectionType = POSTGRES) - void updateLastActivityTimeBulk( - @Define("caseStatements") String caseStatements, - @BindList("nameHashes") List nameHashes); - - @ConnectionAwareSqlQuery( - value = - "SELECT CAST(JSON_EXTRACT(json, '$.lastActivityTime') AS UNSIGNED) as lastActivity " - + "FROM user_entity " - + "WHERE JSON_EXTRACT(json, '$.isBot') = false " - + "AND JSON_EXTRACT(json, '$.lastActivityTime') IS NOT NULL " - + "AND deleted = false " - + "ORDER BY CAST(JSON_EXTRACT(json, '$.lastActivityTime') AS UNSIGNED) DESC " - + "LIMIT 1", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT CAST(json->>'lastActivityTime' AS BIGINT) as lastActivity " - + "FROM user_entity " - + "WHERE (json->>'isBot')::boolean = false " - + "AND json->>'lastActivityTime' IS NOT NULL " - + "AND deleted = false " - + "ORDER BY CAST(json->>'lastActivityTime' AS BIGINT) DESC " - + "LIMIT 1", - connectionType = POSTGRES) - Long getMaxLastActivityTime(); - - @SqlQuery( - "SELECT COUNT(DISTINCT id) FROM user_entity " - + "WHERE isBot = false " - + "AND deleted = false " - + "AND lastActivityTime >= :since") - int countDailyActiveUsers(@Bind("since") long since); - } - - interface ChangeEventDAO { - @SqlQuery( - "SELECT json FROM change_event ce where ce.offset > :offset ORDER BY ce.eventTime DESC LIMIT :limit OFFSET :paginationOffset") - List listUnprocessedEvents( - @Bind("offset") long offset, - @Bind("limit") int limit, - @Bind("paginationOffset") int paginationOffset); - - @SqlQuery( - "SELECT json, source FROM consumers_dlq WHERE id = :id ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") - @RegisterRowMapper(FailedEventResponseMapper.class) - List listFailedEventsById( - @Bind("id") String id, - @Bind("limit") int limit, - @Bind("paginationOffset") int paginationOffset); - - @SqlQuery("SELECT COUNT(*) FROM consumers_dlq WHERE id = :id") - long countFailedEvents(@Bind("id") String id); - - @SqlQuery( - "SELECT json, source FROM consumers_dlq WHERE id = :id AND source = :source ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") - @RegisterRowMapper(FailedEventResponseMapper.class) - List listFailedEventsByIdAndSource( - @Bind("id") String id, - @Bind("source") String source, - @Bind("limit") int limit, - @Bind("paginationOffset") int paginationOffset); - - @SqlQuery( - "SELECT json, source FROM consumers_dlq ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") - @RegisterRowMapper(FailedEventResponseMapper.class) - List listAllFailedEvents( - @Bind("limit") int limit, @Bind("paginationOffset") int paginationOffset); - - @SqlQuery( - "SELECT json, source FROM consumers_dlq WHERE source = :source ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") - @RegisterRowMapper(FailedEventResponseMapper.class) - List listAllFailedEventsBySource( - @Bind("source") String source, - @Bind("limit") int limit, - @Bind("paginationOffset") int paginationOffset); - - @ConnectionAwareSqlQuery( - value = - "SELECT json, status, timestamp " - + "FROM ( " - + " SELECT json, 'FAILED' AS status, timestamp " - + " FROM consumers_dlq WHERE id = :id " - + " UNION ALL " - + " SELECT json, 'SUCCESSFUL' AS status, timestamp " - + " FROM successful_sent_change_events WHERE event_subscription_id = :id " - + ") AS combined_events " - + "ORDER BY timestamp DESC " - + "LIMIT :limit OFFSET :paginationOffset", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json, status, timestamp " - + "FROM ( " - + " SELECT json, 'failed' AS status, timestamp " - + " FROM consumers_dlq WHERE id = :id " - + " UNION ALL " - + " SELECT json, 'successful' AS status, timestamp " - + " FROM successful_sent_change_events WHERE event_subscription_id = :id " - + ") AS combined_events " - + "ORDER BY timestamp DESC " - + "LIMIT :limit OFFSET :paginationOffset", - connectionType = POSTGRES) - @RegisterRowMapper(EventResponseMapper.class) - List listAllEventsWithStatuses( - @Bind("id") String id, - @Bind("limit") int limit, - @Bind("paginationOffset") long paginationOffset); - - @SqlQuery("SELECT json FROM change_event ce where ce.offset > :offset") - List listUnprocessedEvents(@Bind("offset") long offset); - - @SqlQuery( - "SELECT CASE WHEN EXISTS (SELECT 1 FROM event_subscription_entity WHERE id = :id) THEN 1 ELSE 0 END AS record_exists") - int recordExists(@Bind("id") String id); - - @ConnectionAwareSqlUpdate( - value = "INSERT INTO change_event (json) VALUES (:json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "INSERT INTO change_event (json) VALUES (:json :: jsonb)", - connectionType = POSTGRES) - void insert(@Bind("json") String json); - - @Transaction - @ConnectionAwareSqlBatch( - value = "INSERT INTO change_event (json) VALUES (:json)", - connectionType = MYSQL) - @ConnectionAwareSqlBatch( - value = "INSERT INTO change_event (json) VALUES (CAST(:json AS jsonb))", - connectionType = POSTGRES) - void insertBatchRows(@Bind("json") List jsons); - - default void insertBatch(List jsons) { - if (nullOrEmpty(jsons)) { - return; - } - insertBatchRows(jsons); - } - - @SqlUpdate("DELETE FROM change_event WHERE entityType = :entityType") - void deleteAll(@Bind("entityType") String entityType); - - default List list(EventType eventType, List entityTypes, long timestamp) { - if (nullOrEmpty(entityTypes)) { - return Collections.emptyList(); - } - if (entityTypes.get(0).equals("*")) { - return listWithoutEntityFilter(eventType.value(), timestamp); - } - return listWithEntityFilter(eventType.value(), entityTypes, timestamp); - } - - @SqlQuery( - "SELECT json FROM change_event WHERE " - + "eventType = :eventType AND (entityType IN ()) AND eventTime >= :timestamp " - + "ORDER BY eventTime ASC") - List listWithEntityFilter( - @Bind("eventType") String eventType, - @BindList("entityTypes") List entityTypes, - @Bind("timestamp") long timestamp); - - @SqlQuery( - "SELECT json FROM change_event WHERE " - + "eventType = :eventType AND eventTime >= :timestamp " - + "ORDER BY eventTime ASC") - List listWithoutEntityFilter( - @Bind("eventType") String eventType, @Bind("timestamp") long timestamp); - - @SqlQuery( - "SELECT json FROM change_event ce WHERE ce.offset > :offset ORDER BY ce.offset ASC LIMIT :limit") - List list(@Bind("limit") long limit, @Bind("offset") long offset); - - @ConnectionAwareSqlQuery(value = "SELECT MAX(offset) FROM change_event", connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT MAX(\"offset\") FROM change_event", - connectionType = POSTGRES) - long getLatestOffset(); - - @SqlQuery("SELECT count(*) FROM change_event") - long listCount(); - - /** Record holding change event offset and JSON for cursor-based pagination. */ - record ChangeEventRecord(long offset, String json) {} - - /** Returns change events with their offset values for accurate cursor tracking. */ - @ConnectionAwareSqlQuery( - value = - "SELECT `offset`, json FROM change_event WHERE `offset` > :afterOffset ORDER BY `offset` ASC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT \"offset\", json FROM change_event WHERE \"offset\" > :afterOffset ORDER BY \"offset\" ASC LIMIT :limit", - connectionType = POSTGRES) - @RegisterRowMapper(ChangeEventRecordMapper.class) - List listWithOffset( - @Bind("limit") int limit, @Bind("afterOffset") long afterOffset); - } - - class ChangeEventRecordMapper implements RowMapper { - @Override - public ChangeEventDAO.ChangeEventRecord map(ResultSet rs, StatementContext ctx) - throws SQLException { - return new ChangeEventDAO.ChangeEventRecord(rs.getLong("offset"), rs.getString("json")); - } - } - - class FailedEventResponseMapper implements RowMapper { - @Override - public FailedEventResponse map(ResultSet rs, StatementContext ctx) throws SQLException { - FailedEventResponse response = new FailedEventResponse(); - FailedEvent failedEvent = JsonUtils.readValue(rs.getString("json"), FailedEvent.class); - response.setFailingSubscriptionId(failedEvent.getFailingSubscriptionId()); - response.setChangeEvent(failedEvent.getChangeEvent()); - response.setReason(failedEvent.getReason()); - response.setSource(rs.getString("source")); - response.setTimestamp(failedEvent.getTimestamp()); - return response; - } - } - - class EventResponseMapper implements RowMapper { - @Override - public TypedEvent map(ResultSet rs, StatementContext ctx) throws SQLException { - TypedEvent response = new TypedEvent(); - String status = rs.getString("status").toLowerCase(); - - if (TypedEvent.Status.FAILED.value().equalsIgnoreCase(status)) { - FailedEvent failedEvent = JsonUtils.readValue(rs.getString("json"), FailedEvent.class); - response.setData(List.of(failedEvent)); - response.setStatus(TypedEvent.Status.FAILED); - } else { - ChangeEvent changeEvent = JsonUtils.readValue(rs.getString("json"), ChangeEvent.class); - response.setData(List.of(changeEvent)); - response.setStatus(TypedEvent.Status.fromValue(status)); - } - - long timestampMillis = rs.getLong("timestamp"); - response.setTimestamp((double) timestampMillis); - return response; - } - } - - interface TypeEntityDAO extends EntityDAO { - @Override - default String getTableName() { - return "type_entity"; - } - - @Override - default Class getEntityClass() { - return Type.class; - } - - @Override - default boolean supportsSoftDelete() { - return false; - } - } - - interface TestDefinitionDAO extends EntityDAO { - @Override - default String getTableName() { - return "test_definition"; - } - - @Override - default Class getEntityClass() { - return TestDefinition.class; - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String entityType = filter.getQueryParam("entityType"); - String testPlatform = filter.getQueryParam("testPlatform"); - String supportedDataType = filter.getQueryParam("supportedDataType"); - String supportedService = filter.getQueryParam("supportedService"); - String enabled = filter.getQueryParam("enabled"); - String condition = filter.getCondition(); - - if (entityType == null - && testPlatform == null - && supportedDataType == null - && supportedService == null - && enabled == null) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - - StringBuilder mysqlCondition = new StringBuilder(); - StringBuilder psqlCondition = new StringBuilder(); - - mysqlCondition.append(String.format("%s ", condition)); - psqlCondition.append(String.format("%s ", condition)); - - if (testPlatform != null) { - filter.queryParams.put("testPlatformLike", String.format("%%%s%%", testPlatform)); - mysqlCondition.append("AND json_extract(json, '$.testPlatforms') LIKE :testPlatformLike "); - psqlCondition.append("AND json->>'testPlatforms' LIKE :testPlatformLike "); - } - - if (entityType != null) { - mysqlCondition.append("AND entityType=:entityType "); - psqlCondition.append("AND entityType=:entityType "); - } - - if (supportedDataType != null) { - filter.queryParams.put("supportedDataTypeLike", String.format("%%%s%%", supportedDataType)); - mysqlCondition.append( - "AND json_extract(json, '$.supportedDataTypes') LIKE :supportedDataTypeLike "); - psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike "); - } - - if (supportedService != null) { - filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); - mysqlCondition.append( - "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " - + "OR json_extract(json, '$.supportedServices') IS NULL " - + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); - psqlCondition.append( - "AND (json->>'supportedServices' = '[]' " - + "OR json->>'supportedServices' IS NULL " - + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); - } - - if (enabled != null) { - String enabledValue = Boolean.parseBoolean(enabled) ? "TRUE" : "FALSE"; - mysqlCondition.append("AND enabled=").append(enabledValue).append(" "); - psqlCondition.append("AND enabled=").append(enabledValue).append(" "); - } - - return listBefore( - getTableName(), - filter.getQueryParams(), - mysqlCondition.toString(), - psqlCondition.toString(), - limit, - beforeName, - beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String entityType = filter.getQueryParam("entityType"); - String testPlatform = filter.getQueryParam("testPlatform"); - String supportedDataType = filter.getQueryParam("supportedDataType"); - String supportedService = filter.getQueryParam("supportedService"); - String enabled = filter.getQueryParam("enabled"); - String condition = filter.getCondition(); - - if (entityType == null - && testPlatform == null - && supportedDataType == null - && supportedService == null - && enabled == null) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - - StringBuilder mysqlCondition = new StringBuilder(); - StringBuilder psqlCondition = new StringBuilder(); - - mysqlCondition.append(String.format("%s ", condition)); - psqlCondition.append(String.format("%s ", condition)); - - if (testPlatform != null) { - filter.queryParams.put("testPlatformLike", String.format("%%%s%%", testPlatform)); - mysqlCondition.append("AND json_extract(json, '$.testPlatforms') LIKE :testPlatformLike "); - psqlCondition.append("AND json->>'testPlatforms' LIKE :testPlatformLike "); - } - - if (entityType != null) { - mysqlCondition.append("AND entityType = :entityType "); - psqlCondition.append("AND entityType = :entityType "); - } - - if (supportedDataType != null) { - filter.queryParams.put("supportedDataTypeLike", String.format("%%%s%%", supportedDataType)); - mysqlCondition.append( - "AND json_extract(json, '$.supportedDataTypes') LIKE :supportedDataTypeLike "); - psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike "); - } - - if (supportedService != null) { - filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); - mysqlCondition.append( - "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " - + "OR json_extract(json, '$.supportedServices') IS NULL " - + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); - psqlCondition.append( - "AND (json->>'supportedServices' = '[]' " - + "OR json->>'supportedServices' IS NULL " - + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); - } - - if (enabled != null) { - String enabledValue = Boolean.parseBoolean(enabled) ? "TRUE" : "FALSE"; - mysqlCondition.append("AND enabled=").append(enabledValue).append(" "); - psqlCondition.append("AND enabled=").append(enabledValue).append(" "); - } - - return listAfter( - getTableName(), - filter.getQueryParams(), - mysqlCondition.toString(), - psqlCondition.toString(), - limit, - afterName, - afterId); - } - - @Override - default int listCount(ListFilter filter) { - String entityType = filter.getQueryParam("entityType"); - String testPlatform = filter.getQueryParam("testPlatform"); - String supportedDataType = filter.getQueryParam("supportedDataType"); - String supportedService = filter.getQueryParam("supportedService"); - String enabled = filter.getQueryParam("enabled"); - String condition = filter.getCondition(); - - if (entityType == null - && testPlatform == null - && supportedDataType == null - && supportedService == null - && enabled == null) { - return EntityDAO.super.listCount(filter); - } - - StringBuilder mysqlCondition = new StringBuilder(); - StringBuilder psqlCondition = new StringBuilder(); - - mysqlCondition.append(String.format("%s ", condition)); - psqlCondition.append(String.format("%s ", condition)); - - if (testPlatform != null) { - filter.queryParams.put("testPlatformLike", String.format("%%%s%%", testPlatform)); - mysqlCondition.append("AND json_extract(json, '$.testPlatforms') LIKE :testPlatformLike "); - psqlCondition.append("AND json->>'testPlatforms' LIKE :testPlatformLike "); - } - - if (entityType != null) { - mysqlCondition.append("AND entityType=:entityType "); - psqlCondition.append("AND entityType=:entityType "); - } - - if (supportedDataType != null) { - filter.queryParams.put("supportedDataTypeLike", String.format("%%%s%%", supportedDataType)); - mysqlCondition.append( - "AND json_extract(json, '$.supportedDataTypes') LIKE :supportedDataTypeLike "); - psqlCondition.append("AND json->>'supportedDataTypes' LIKE :supportedDataTypeLike "); - } - - if (supportedService != null) { - filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); - mysqlCondition.append( - "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " - + "OR json_extract(json, '$.supportedServices') IS NULL " - + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); - psqlCondition.append( - "AND (json->>'supportedServices' = '[]' " - + "OR json->>'supportedServices' IS NULL " - + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); - } - - if (enabled != null) { - String enabledValue = Boolean.parseBoolean(enabled) ? "TRUE" : "FALSE"; - mysqlCondition.append("AND enabled=").append(enabledValue).append(" "); - psqlCondition.append("AND enabled=").append(enabledValue).append(" "); - } - - return listCount( - getTableName(), - filter.getQueryParams(), - getNameHashColumn(), - mysqlCondition.toString(), - psqlCondition.toString()); - } - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT name, id, json FROM
AND " - + "(
.name < :beforeName OR (
.name = :beforeName AND
.id < :beforeId)) " - + "ORDER BY name DESC,id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT name, id, json FROM
AND " - + "(
.name < :beforeName OR (
.name = :beforeName AND
.id < :beforeId)) " - + "ORDER BY name DESC,id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id", - connectionType = POSTGRES) - List listBefore( - @Define("table") String table, - @BindMap Map params, - @Define("mysqlCond") String mysqlCond, - @Define("psqlCond") String psqlCond, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit", - connectionType = POSTGRES) - List listAfter( - @Define("table") String table, - @BindMap Map params, - @Define("mysqlCond") String mysqlCond, - @Define("psqlCond") String psqlCond, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId); - - @ConnectionAwareSqlQuery( - value = "SELECT count() FROM
", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT count(*) FROM
", - connectionType = POSTGRES) - int listCount( - @Define("table") String table, - @BindMap Map params, - @Define("nameHashColumn") String nameHashColumn, - @Define("mysqlCond") String mysqlCond, - @Define("psqlCond") String psqlCond); - } - - interface TestSuiteDAO extends EntityDAO { - @Override - default String getTableName() { - return "test_suite"; - } - - @Override - default Class getEntityClass() { - return TestSuite.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default int listCount(ListFilter filter) { - String mySqlCondition = filter.getCondition(getTableName()); - String postgresCondition = filter.getCondition(getTableName()); - boolean includeEmptyTestSuite = - Boolean.parseBoolean(filter.getQueryParam("includeEmptyTestSuites")); - if (!includeEmptyTestSuite) { - String condition = - String.format( - "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", - getTableName(), CONTAINS.ordinal(), Entity.TEST_CASE); - mySqlCondition = condition; - postgresCondition = condition; - - mySqlCondition = - String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); - postgresCondition = - String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); - } - return listCountDistinct( - getTableName(), - mySqlCondition, - postgresCondition, - String.format("%s.%s", getTableName(), getNameHashColumn())); - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String mySqlCondition = filter.getCondition(getTableName()); - String postgresCondition = filter.getCondition(getTableName()); - String groupBy = ""; - boolean includeEmptyTestSuite = - Boolean.parseBoolean(filter.getQueryParam("includeEmptyTestSuites")); - if (!includeEmptyTestSuite) { - groupBy = - String.format( - "group by %s.json, %s.name, %s.id", getTableName(), getTableName(), getTableName()); - String condition = - String.format( - "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", - getTableName(), CONTAINS.ordinal(), Entity.TEST_CASE); - mySqlCondition = condition; - postgresCondition = condition; - mySqlCondition = - String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); - postgresCondition = - String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); - } - return listBefore( - getTableName(), mySqlCondition, postgresCondition, limit, beforeName, beforeId, groupBy); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String mySqlCondition = filter.getCondition(getTableName()); - String postgresCondition = filter.getCondition(getTableName()); - String groupBy = ""; - boolean includeEmptyTestSuite = - Boolean.parseBoolean(filter.getQueryParam("includeEmptyTestSuites")); - if (!includeEmptyTestSuite) { - groupBy = - String.format( - "group by %s.json, %s.name, %s.id", getTableName(), getTableName(), getTableName()); - String condition = - String.format( - "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", - getTableName(), CONTAINS.ordinal(), Entity.TEST_CASE); - mySqlCondition = condition; - postgresCondition = condition; - - mySqlCondition = - String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); - postgresCondition = - String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); - } - return listAfter( - getTableName(), mySqlCondition, postgresCondition, limit, afterName, afterId, groupBy); - } - - @SqlQuery( - "SELECT json FROM
tn\n" - + "INNER JOIN (SELECT DISTINCT fromId FROM entity_relationship er\n" - + " AND toEntity = 'testSuite' and fromEntity = :entityType) er ON fromId = tn.id\n" - + "LIMIT :limit OFFSET :offset;") - List listEntitiesWithTestSuite( - @Define("table") String table, - @BindMap Map params, - @Define("cond") String cond, - @Bind("entityType") String entityType, - @Bind("limit") int limit, - @Bind("offset") int offset); - - default List listEntitiesWithTestsuite( - ListFilter filter, String table, String entityType, int limit, int offset) { - return listEntitiesWithTestSuite( - table, filter.getQueryParams(), filter.getCondition(), entityType, limit, offset); - } - - @SqlQuery( - "SELECT COUNT(DISTINCT fromId) FROM entity_relationship er\n" - + " AND toEntity = 'testSuite' and fromEntity = :entityType;") - Integer countEntitiesWithTestSuite( - @BindMap Map params, - @Define("cond") String cond, - @Bind("entityType") String entityType); - - default Integer countEntitiesWithTestsuite(ListFilter filter, String entityType) { - return countEntitiesWithTestSuite(filter.getQueryParams(), filter.getCondition(), entityType); - } - } - - interface TestCaseDAO extends EntityDAO { - @Override - default String getTableName() { - return "test_case"; - } - - @Override - default Class getEntityClass() { - return TestCase.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - default int countOfTestCases(List testCaseIds) { - return countOfTestCases(getTableName(), testCaseIds.stream().map(Object::toString).toList()); - } - - @SqlQuery("SELECT count(*) FROM
WHERE id IN ()") - int countOfTestCases( - @Define("table") String table, @BindList("testCaseIds") List testCaseIds); - - /** - * Returns ids of test cases whose entityFQN equals {@code entityFQN} (table-level tests) or - * starts with {@code entityFQNPrefix} (column-level tests). The prefix must already have LIKE - * metacharacters escaped — callers should route through - * {@link org.openmetadata.service.util.LikeEscape#escape(String)} and append {@code ".%"}. - * Uses {@code ESCAPE '!'} to match the convention used elsewhere in this DAO; backslash is - * unsafe (MySQL treats it as a string-literal escape and JDBI's ColonPrefixSqlParser - * mishandles literal {@code '\'} inside single-quoted SQL strings). - */ - @SqlQuery( - "SELECT id FROM test_case WHERE entityFQN = :entityFQN " - + "OR entityFQN LIKE :entityFQNPrefix ESCAPE '!'") - List findIdsByEntityFQN( - @Bind("entityFQN") String entityFQN, @Bind("entityFQNPrefix") String entityFQNPrefix); - - class TestCaseRecord { - @Getter String json; - @Getter Integer rank; - - public TestCaseRecord(String json, Integer rank) { - this.json = json; - this.rank = rank; - } - } - - class TestCaseRecordMapper implements RowMapper { - @Override - public TestCaseRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new TestCaseRecord(rs.getString("json"), rs.getInt("ranked")); - } - } - } - - interface WebAnalyticEventDAO extends EntityDAO { - @Override - default String getTableName() { - return "web_analytic_event"; - } - - @Override - default Class getEntityClass() { - return WebAnalyticEvent.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface DataInsightCustomChartDAO extends EntityDAO { - @Override - default String getTableName() { - return "di_chart_entity"; - } - - @Override - default Class getEntityClass() { - return DataInsightCustomChart.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface DataInsightChartDAO extends EntityDAO { - @Override - default String getTableName() { - return "data_insight_chart"; - } - - @Override - default Class getEntityClass() { - return DataInsightChart.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface EntityExtensionTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "entity_extension_time_series"; - } - - @ConnectionAwareSqlQuery( - value = - "SELECT " - + " DATE(FROM_UNIXTIME(eets.timestamp / 1000)) as date_key, " - + " JSON_UNQUOTE(JSON_EXTRACT(eets.json, '$.executionStatus')) as status, " - + " COUNT(*) as count " - + "FROM entity_extension_time_series eets " - + "INNER JOIN pipeline_entity pe ON eets.entityFQNHash = pe.fqnHash " - + "WHERE eets.extension = 'pipeline.pipelineStatus' " - + " AND pe.deleted = 0 " - + " AND eets.timestamp >= :startTs " - + " AND eets.timestamp <= :endTs " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + "GROUP BY date_key, status " - + "ORDER BY date_key ASC", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT " - + " DATE(TO_TIMESTAMP(eets.timestamp / 1000)) as date_key, " - + " eets.json->>'executionStatus' as status, " - + " COUNT(*) as count " - + "FROM entity_extension_time_series eets " - + "INNER JOIN pipeline_entity pe ON eets.entityFQNHash = pe.fqnHash " - + "WHERE eets.extension = 'pipeline.pipelineStatus' " - + " AND pe.deleted = false " - + " AND eets.timestamp >= :startTs " - + " AND eets.timestamp <= :endTs " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + "GROUP BY date_key, status " - + "ORDER BY date_key ASC", - connectionType = POSTGRES) - @RegisterRowMapper(ExecutionTrendRowMapper.class) - List getExecutionTrendData( - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs, - @Define("pipelineFqnFilter") String pipelineFqnFilter, - @Define("serviceTypeFilter") String serviceTypeFilter, - @Define("serviceFilter") String serviceFilter, - @Define("mysqlStatusFilter") String mysqlStatusFilter, - @Define("postgresStatusFilter") String postgresStatusFilter, - @Define("domainFilter") String domainFilter, - @Define("ownerFilter") String ownerFilter, - @Define("tierFilter") String tierFilter); - - @ConnectionAwareSqlQuery( - value = - "WITH runtime_calc AS ( " - + " SELECT " - + " eets.*, " - + " pe.fqnHash, " - + " CASE " - + " WHEN JSON_LENGTH(JSON_EXTRACT(eets.json, '$.taskStatus')) > 0 " - + " AND JSON_EXTRACT(eets.json, '$.taskStatus[0].endTime') IS NOT NULL THEN " - + " ( " - + " SELECT MAX(CAST(JSON_EXTRACT(task.value, '$.endTime') AS UNSIGNED)) " - + " FROM JSON_TABLE(eets.json, '$.taskStatus[*]' COLUMNS(value JSON PATH '$')) AS task " - + " WHERE JSON_EXTRACT(task.value, '$.endTime') IS NOT NULL " - + " ) - ( " - + " SELECT MIN(CAST(JSON_EXTRACT(task.value, '$.startTime') AS UNSIGNED)) " - + " FROM JSON_TABLE(eets.json, '$.taskStatus[*]' COLUMNS(value JSON PATH '$')) AS task " - + " WHERE JSON_EXTRACT(task.value, '$.startTime') IS NOT NULL " - + " ) " - + " WHEN JSON_EXTRACT(eets.json, '$.endTime') IS NOT NULL THEN " - + " JSON_EXTRACT(eets.json, '$.endTime') - eets.timestamp " - + " ELSE NULL " - + " END AS runtime " - + " FROM entity_extension_time_series eets " - + " INNER JOIN pipeline_entity pe ON eets.entityFQNHash = pe.fqnHash " - + " WHERE eets.extension = 'pipeline.pipelineStatus' " - + " AND pe.deleted = 0 " - + " AND eets.timestamp >= :startTs " - + " AND eets.timestamp <= :endTs " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + ") " - + "SELECT " - + " DATE(FROM_UNIXTIME(timestamp / 1000)) as date_key, " - + " MIN(timestamp) as first_timestamp, " - + " MAX(runtime) as max_runtime, " - + " MIN(runtime) as min_runtime, " - + " AVG(runtime) as avg_runtime, " - + " COUNT(DISTINCT fqnHash) as total_pipelines " - + "FROM runtime_calc " - + "WHERE runtime IS NOT NULL " - + "GROUP BY date_key " - + "ORDER BY date_key ASC", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "WITH runtime_calc AS ( " - + " SELECT " - + " eets.timestamp, " - + " eets.json, " - + " pe.fqnHash, " - + " CASE " - + " WHEN jsonb_array_length(COALESCE(eets.json->'taskStatus', '[]'::jsonb)) > 0 " - + " AND EXISTS ( " - + " SELECT 1 FROM jsonb_array_elements(eets.json->'taskStatus') AS task " - + " WHERE task->>'endTime' IS NOT NULL " - + " ) THEN " - + " ( " - + " SELECT MAX((task->>'endTime')::bigint) " - + " FROM jsonb_array_elements(eets.json->'taskStatus') AS task " - + " WHERE task->>'endTime' IS NOT NULL " - + " ) - ( " - + " SELECT MIN((task->>'startTime')::bigint) " - + " FROM jsonb_array_elements(eets.json->'taskStatus') AS task " - + " WHERE task->>'startTime' IS NOT NULL " - + " ) " - + " WHEN eets.json->>'endTime' IS NOT NULL THEN " - + " (eets.json->>'endTime')::bigint - eets.timestamp " - + " ELSE NULL " - + " END AS runtime " - + " FROM entity_extension_time_series eets " - + " INNER JOIN pipeline_entity pe ON eets.entityFQNHash = pe.fqnHash " - + " WHERE eets.extension = 'pipeline.pipelineStatus' " - + " AND pe.deleted = false " - + " AND eets.timestamp >= :startTs " - + " AND eets.timestamp <= :endTs " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + ") " - + "SELECT " - + " DATE(TO_TIMESTAMP(timestamp / 1000)) as date_key, " - + " MIN(timestamp) as first_timestamp, " - + " MAX(runtime) as max_runtime, " - + " MIN(runtime) as min_runtime, " - + " AVG(runtime) as avg_runtime, " - + " COUNT(DISTINCT fqnHash) as total_pipelines " - + "FROM runtime_calc " - + "WHERE runtime IS NOT NULL " - + "GROUP BY date_key " - + "ORDER BY date_key ASC", - connectionType = POSTGRES) - @RegisterRowMapper(RuntimeTrendRowMapper.class) - List getRuntimeTrendData( - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs, - @Define("pipelineFqnFilter") String pipelineFqnFilter, - @Define("serviceTypeFilter") String serviceTypeFilter, - @Define("serviceFilter") String serviceFilter, - @Define("mysqlStatusFilter") String mysqlStatusFilter, - @Define("postgresStatusFilter") String postgresStatusFilter, - @Define("domainFilter") String domainFilter, - @Define("ownerFilter") String ownerFilter, - @Define("tierFilter") String tierFilter); - - @ConnectionAwareSqlQuery( - value = - "SELECT " - + " JSON_UNQUOTE(JSON_EXTRACT(pe.json, '$.serviceType')) as service_type, " - + " COUNT(*) as pipeline_count " - + "FROM pipeline_entity pe " - + "LEFT JOIN entity_extension_time_series eets " - + " ON pe.fqnHash = eets.entityFQNHash " - + " AND eets.extension = 'pipeline.pipelineStatus' " - + "WHERE pe.deleted = 0 " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + "GROUP BY service_type", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT " - + " pe.json->>'serviceType' as service_type, " - + " COUNT(*) as pipeline_count " - + "FROM pipeline_entity pe " - + "LEFT JOIN entity_extension_time_series eets " - + " ON pe.fqnHash = eets.entityFQNHash " - + " AND eets.extension = 'pipeline.pipelineStatus' " - + "WHERE pe.deleted = false " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + "GROUP BY service_type", - connectionType = POSTGRES) - @RegisterRowMapper(ServiceBreakdownRowMapper.class) - List getServiceBreakdown( - @Define("serviceTypeFilter") String serviceTypeFilter, - @Define("serviceFilter") String serviceFilter, - @Define("mysqlStatusFilter") String mysqlStatusFilter, - @Define("postgresStatusFilter") String postgresStatusFilter, - @Define("domainFilter") String domainFilter, - @Define("ownerFilter") String ownerFilter, - @Define("tierFilter") String tierFilter, - @Define("startTsFilter") String startTsFilter, - @Define("endTsFilter") String endTsFilter); - - @ConnectionAwareSqlQuery( - value = - "SELECT " - + " COUNT(DISTINCT pe.fqnHash) as total_pipelines, " - + " COUNT(DISTINCT CASE WHEN eets.entityFQNHash IS NOT NULL THEN pe.fqnHash END) as active_pipelines, " - + " COUNT(DISTINCT CASE WHEN JSON_UNQUOTE(JSON_EXTRACT(eets.json, '$.executionStatus')) = 'Successful' THEN pe.fqnHash END) as successful_pipelines, " - + " COUNT(DISTINCT CASE WHEN JSON_UNQUOTE(JSON_EXTRACT(eets.json, '$.executionStatus')) = 'Failed' THEN pe.fqnHash END) as failed_pipelines " - + "FROM pipeline_entity pe " - + "LEFT JOIN entity_extension_time_series eets " - + " ON pe.fqnHash = eets.entityFQNHash " - + " AND eets.extension = 'pipeline.pipelineStatus' " - + "WHERE pe.deleted = 0 " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + " ", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT " - + " COUNT(DISTINCT pe.fqnHash) as total_pipelines, " - + " COUNT(DISTINCT CASE WHEN eets.entityFQNHash IS NOT NULL THEN pe.fqnHash END) as active_pipelines, " - + " COUNT(DISTINCT CASE WHEN eets.json->>'executionStatus' = 'Successful' THEN pe.fqnHash END) as successful_pipelines, " - + " COUNT(DISTINCT CASE WHEN eets.json->>'executionStatus' = 'Failed' THEN pe.fqnHash END) as failed_pipelines " - + "FROM pipeline_entity pe " - + "LEFT JOIN entity_extension_time_series eets " - + " ON pe.fqnHash = eets.entityFQNHash " - + " AND eets.extension = 'pipeline.pipelineStatus' " - + "WHERE pe.deleted = false " - + " " - + " " - + " " - + " " - + " " - + " " - + " " - + " ", - connectionType = POSTGRES) - @RegisterRowMapper(PipelineMetricsRowMapper.class) - PipelineMetricsRow getPipelineMetricsData( - @Define("serviceTypeFilter") String serviceTypeFilter, - @Define("serviceFilter") String serviceFilter, - @Define("mysqlStatusFilter") String mysqlStatusFilter, - @Define("postgresStatusFilter") String postgresStatusFilter, - @Define("domainFilter") String domainFilter, - @Define("ownerFilter") String ownerFilter, - @Define("tierFilter") String tierFilter, - @Define("startTsFilter") String startTsFilter, - @Define("endTsFilter") String endTsFilter); - - @ConnectionAwareSqlQuery( - value = - "SELECT pe.id, pe.json, " - + "(SELECT eets_inner.json FROM entity_extension_time_series eets_inner " - + " WHERE eets_inner.entityFQNHash = pe.fqnHash " - + " AND eets_inner.extension = 'pipeline.pipelineStatus' " - + " ORDER BY eets_inner.timestamp DESC LIMIT 1) as latest_status " - + "FROM pipeline_entity pe " - + "WHERE pe.deleted = 0 " - + " " - + " " - + " " - + " " - + " " - + " AND (:search IS NULL OR pe.name LIKE CONCAT('%', :search, '%') OR JSON_UNQUOTE(JSON_EXTRACT(pe.json, '$.fullyQualifiedName')) LIKE CONCAT('%', :search, '%')) " - + " " - + "ORDER BY pe.name " - + "LIMIT :limit OFFSET :offset", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT pe.id, pe.json, " - + "(SELECT eets_inner.json FROM entity_extension_time_series eets_inner " - + " WHERE eets_inner.entityFQNHash = pe.fqnHash " - + " AND eets_inner.extension = 'pipeline.pipelineStatus' " - + " ORDER BY eets_inner.timestamp DESC LIMIT 1) as latest_status " - + "FROM pipeline_entity pe " - + "WHERE pe.deleted = false " - + " " - + " " - + " " - + " " - + " " - + " AND (:search IS NULL OR pe.name LIKE '%' || :search || '%' OR pe.json->>'fullyQualifiedName' LIKE '%' || :search || '%') " - + " " - + "ORDER BY pe.name " - + "LIMIT :limit OFFSET :offset", - connectionType = POSTGRES) - @RegisterRowMapper(PipelineSummaryRowMapper.class) - List listPipelineSummariesFiltered( - @Define("serviceFilter") String serviceFilter, - @Define("mysqlServiceTypeFilter") String mysqlServiceTypeFilter, - @Define("postgresServiceTypeFilter") String postgresServiceTypeFilter, - @Define("domainFilter") String domainFilter, - @Define("ownerFilter") String ownerFilter, - @Define("tierFilter") String tierFilter, - @Define("mysqlStatusFilter") String mysqlStatusFilter, - @Define("postgresStatusFilter") String postgresStatusFilter, - @Bind("search") String search, - @Bind("limit") int limit, - @Bind("offset") int offset); - - @ConnectionAwareSqlQuery( - value = - "SELECT COUNT(DISTINCT pe.id) " - + "FROM pipeline_entity pe " - + "WHERE pe.deleted = 0 " - + " " - + " " - + " " - + " " - + " " - + " AND (:search IS NULL OR pe.name LIKE CONCAT('%', :search, '%') OR JSON_UNQUOTE(JSON_EXTRACT(pe.json, '$.fullyQualifiedName')) LIKE CONCAT('%', :search, '%')) " - + " ", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT COUNT(DISTINCT pe.id) " - + "FROM pipeline_entity pe " - + "WHERE pe.deleted = false " - + " " - + " " - + " " - + " " - + " " - + " AND (:search IS NULL OR pe.name LIKE '%' || :search || '%' OR pe.json->>'fullyQualifiedName' LIKE '%' || :search || '%') " - + " ", - connectionType = POSTGRES) - int countPipelineSummariesFiltered( - @Define("serviceFilter") String serviceFilter, - @Define("mysqlServiceTypeFilter") String mysqlServiceTypeFilter, - @Define("postgresServiceTypeFilter") String postgresServiceTypeFilter, - @Define("domainFilter") String domainFilter, - @Define("ownerFilter") String ownerFilter, - @Define("tierFilter") String tierFilter, - @Define("mysqlStatusFilter") String mysqlStatusFilter, - @Define("postgresStatusFilter") String postgresStatusFilter, - @Bind("search") String search); - } - - interface AppsDataStore { - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json) ON DUPLICATE KEY UPDATE json = VALUES(json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json :: jsonb) ON CONFLICT (identifier, type) DO UPDATE SET json = EXCLUDED.json", - connectionType = POSTGRES) - void insert( - @Bind("identifier") String identifier, - @Bind("type") String type, - @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_data_store set json = :json where identifier = :identifier AND type=:type", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_data_store set json = (:json :: jsonb) where identifier = :identifier AND type=:type", - connectionType = POSTGRES) - void update( - @Bind("identifier") String identifier, - @Bind("type") String type, - @Bind("json") String json); - - @SqlUpdate("DELETE FROM apps_data_store WHERE identifier = :identifier AND type = :type") - void delete(@Bind("identifier") String identifier, @Bind("type") String type); - - @SqlQuery( - "SELECT count(*) FROM apps_data_store where identifier = :identifier AND type = :type") - int listAppDataCount(@Bind("identifier") String identifier, @Bind("type") String type); - - @SqlQuery( - "SELECT json FROM apps_data_store where identifier in () AND type = :type") - List listAppsDataWithIds( - @BindList("identifier") List identifier, @Bind("type") String type); - - @SqlQuery("SELECT json FROM apps_data_store where type = :type") - List listAppsDataWithType(@Bind("type") String type); - - @SqlQuery("SELECT json FROM apps_data_store where identifier = :identifier AND type = :type") - String findAppData(@Bind("identifier") String identifier, @Bind("type") String type); - } - - interface AppExtensionTimeSeries { - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO apps_extension_time_series(json, extension) VALUES (:json, :extension)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO apps_extension_time_series(json, extension) VALUES (:json :: jsonb, :extension)", - connectionType = POSTGRES) - void insert(@Bind("json") String json, @Bind("extension") String extension); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'stopped') where appId=:appId AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status'", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"stopped\"') WHERE appId = :appId AND json->>'status' = 'running' AND extension = 'status'", - connectionType = POSTGRES) - void markStaleEntriesStopped(@Bind("appId") String appId); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'stopped') WHERE appName=:appName AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status'", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"stopped\"') WHERE appName = :appName AND json->>'status' = 'running' AND extension = 'status'", - connectionType = POSTGRES) - void markStaleEntriesStoppedByName(@Bind("appName") String appName); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'stopped') WHERE appName=:appName AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status' AND timestamp < :beforeTimestamp", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"stopped\"') WHERE appName = :appName AND json->>'status' = 'running' AND extension = 'status' AND timestamp < :beforeTimestamp", - connectionType = POSTGRES) - void markStaleEntriesStoppedBefore( - @Bind("appName") String appName, @Bind("beforeTimestamp") long beforeTimestamp); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'failed') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status'", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"failed\"') WHERE json->>'status' = 'running' AND extension = 'status'", - connectionType = POSTGRES) - void markAllStaleEntriesFailed(); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'failed') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status' AND appName != :appName", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"failed\"') WHERE json->>'status' = 'running' AND extension = 'status' AND appName != :appName", - connectionType = POSTGRES) - void markAllStaleEntriesFailedExcludingApp(@Bind("appName") String appName); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'failed') WHERE appName=:appName AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status'", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"failed\"') WHERE appName = :appName AND json->>'status' = 'running' AND extension = 'status'", - connectionType = POSTGRES) - void markRunningEntriesFailedByName(@Bind("appName") String appName); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'running') WHERE appId = :appId AND extension = 'status' AND timestamp = :timestamp", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"running\"') WHERE appId = :appId AND extension = 'status' AND timestamp = :timestamp", - connectionType = POSTGRES) - void markEntryRunning(@Bind("appId") String appId, @Bind("timestamp") long timestamp); - - @SqlQuery( - "SELECT json FROM apps_extension_time_series WHERE appId = :appId AND extension = :extension AND timestamp = :timestamp") - String getByAppIdAndTimestamp( - @Bind("appId") String appId, - @Bind("timestamp") long timestamp, - @Bind("extension") String extension); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series set json = :json where appId=:appId and timestamp=:timestamp and extension=:extension", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE apps_extension_time_series set json = (:json :: jsonb) where appId=:appId and timestamp=:timestamp and extension=:extension", - connectionType = POSTGRES) - void update( - @Bind("appId") String appId, - @Bind("json") String json, - @Bind("timestamp") Long timestamp, - @Bind("extension") String extension); - - @SqlUpdate( - "DELETE FROM apps_extension_time_series WHERE appId = :appId AND extension = :extension") - void delete(@Bind("appId") String appId, @Bind("extension") String extension); - - @SqlUpdate("DELETE FROM apps_extension_time_series WHERE appId = :appId") - void deleteAllByAppId(@Bind("appId") String appId); - - @SqlQuery( - "SELECT count(*) FROM apps_extension_time_series where appId = :appId and extension = :extension AND ") - int listAppExtensionCount( - @Bind("appId") String appId, - @Bind("extension") String extension, - @BindJsonContains(value = "service_filter", path = "$.services", property = "id") - UUID service); - - @SqlQuery( - "SELECT count(*) FROM apps_extension_time_series where appId = :appId and extension = :extension AND timestamp > :startTime AND ") - int listAppExtensionCountAfterTime( - @Bind("appId") String appId, - @Bind("startTime") long startTime, - @Bind("extension") String extension, - @BindJsonContains( - value = "service_filter", - path = "$.services", - property = "id", - ifNull = "TRUE") - UUID service); - - @SqlQuery( - "SELECT json FROM apps_extension_time_series where appId = :appId AND extension = :extension AND ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") - List listAppExtension( - @Bind("appId") String appId, - @Bind("limit") int limit, - @Bind("offset") int offset, - @Bind("extension") String extension, - @BindJsonContains( - value = "service_filter", - path = "$.services", - property = "id", - ifNull = "TRUE") - UUID service); - - @SqlQuery( - "SELECT json FROM apps_extension_time_series where appId = :appId AND extension = :extension AND timestamp > :startTime AND ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") - List listAppExtensionAfterTime( - @Bind("appId") String appId, - @Bind("limit") int limit, - @Bind("offset") int offset, - @Bind("startTime") long startTime, - @Bind("extension") String extension, - @BindJsonContains( - value = "service_filter", - path = "$.services", - property = "id", - ifNull = "TRUE") - UUID service); - - // Prepare methods to get extension by name instead of ID - // For example, for limits we need to fetch by app name to ensure if we reinstall the app, - // they'll still be taken into account - @SqlQuery( - "SELECT count(*) FROM apps_extension_time_series where appName = :appName and extension = :extension") - int listAppExtensionCountByName( - @Bind("appName") String appName, @Bind("extension") String extension); - - @SqlQuery( - "SELECT count(*) FROM apps_extension_time_series where appName = :appName and extension = :extension AND timestamp > :startTime") - int listAppExtensionCountAfterTimeByName( - @Bind("appName") String appName, - @Bind("startTime") long startTime, - @Bind("extension") String extension); - - @SqlQuery( - "SELECT json FROM apps_extension_time_series where appName = :appName AND extension = :extension ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") - List listAppExtensionByName( - @Bind("appName") String appName, - @Bind("limit") int limit, - @Bind("offset") int offset, - @Bind("extension") String extension); - - @SqlQuery( - "SELECT json FROM apps_extension_time_series where appName = :appName AND extension = :extension AND timestamp > :startTime ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") - List listAppExtensionAfterTimeByName( - @Bind("appName") String appName, - @Bind("limit") int limit, - @Bind("offset") int offset, - @Bind("startTime") long startTime, - @Bind("extension") String extension); - - @SqlQuery( - "SELECT json FROM apps_extension_time_series where appName = :appName AND extension = :extension AND timestamp >= :startTime AND timestamp < :endTime ORDER BY timestamp ASC LIMIT :limit OFFSET :offset") - List listAppExtensionInWindowByName( - @Bind("appName") String appName, - @Bind("limit") int limit, - @Bind("offset") int offset, - @Bind("startTime") long startTime, - @Bind("endTime") long endTime, - @Bind("extension") String extension); - - default List listAppExtensionAfterTime( - String appId, int limit, int offset, long startTime, String extension) { - return listAppExtensionAfterTime(appId, limit, offset, startTime, extension, null); - } - - default int listAppExtensionCountAfterTime(String appName, long startTime, String extension) { - return listAppExtensionCountAfterTime(appName, startTime, extension, null); - } - - default List listAppExtension(String appName, int limit, int offset, String extension) { - return listAppExtension(appName, limit, offset, extension, null); - } - } - - interface ReportDataTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "report_data_time_series"; - } - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType and date = :date", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType and DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) = DATE(:date)", - connectionType = POSTGRES) - void deleteReportDataTypeAtDate( - @BindFQN("reportDataType") String reportDataType, @Bind("date") String date); - - @SqlUpdate("DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType") - void deletePreviousReportData(@BindFQN("reportDataType") String reportDataType); - } - - interface ProfilerDataTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "profiler_data_time_series"; - } - - @SqlQuery( - "SELECT p.entityFQNHash, p.json " - + "FROM
p " - + "JOIN (" - + " SELECT entityFQNHash, MAX(timestamp) AS latestTs " - + " FROM
" - + " WHERE entityFQNHash IN () AND extension = :extension " - + " GROUP BY entityFQNHash" - + ") latest " - + "ON p.entityFQNHash = latest.entityFQNHash AND p.timestamp = latest.latestTs " - + "WHERE p.extension = :extension " - + "AND p.entityFQNHash IN ()") - @RegisterRowMapper(LatestExtensionRecordMapper.class) - List getLatestExtensionsBatch( - @Define("table") String table, - @BindListFQN("entityFQNHashes") List entityFQNHashes, - @Bind("extension") String extension); - - default List getLatestExtensionsBatch( - List entityFQNHashes, String extension) { - if (entityFQNHashes == null || entityFQNHashes.isEmpty()) { - return Collections.emptyList(); - } - return getLatestExtensionsBatch(getTimeSeriesTableName(), entityFQNHashes, extension); - } - - @SqlQuery( - "SELECT json FROM
" - + "AND timestamp >= :startTs and timestamp <= :endTs ORDER BY timestamp DESC") - List listEntityProfileAtTimestamp( - @Define("table") String table, - @BindMap Map params, - @Define("cond") String cond, - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs); - - default List listEntityProfileData(ListFilter filter, Long startTs, Long endTs) { - return listEntityProfileAtTimestamp( - getTimeSeriesTableName(), filter.getQueryParams(), filter.getCondition(), startTs, endTs); - } - - @SqlUpdate("DELETE FROM
AND timestamp = :timestamp") - void deleteEntityProfileData( - @Define("table") String table, - @BindMap Map params, - @Define("cond") String cond, - @Bind("timestamp") Long timestamp); - - default void deleteEntityProfileData(ListFilter filter, Long timestamp) { - deleteEntityProfileData( - getTimeSeriesTableName(), filter.getQueryParams(), filter.getCondition(), timestamp); - } - - // profiler_data_time_series has no id column (unique key is - // entityFQNHash + extension + operation + timestamp), so we limit by - // row count using single-table DELETE+LIMIT on MySQL and ctid IN (...) on Postgres. - // This bounds the rows deleted per batch, matching the other orphan-cleanup queries. - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM profiler_data_time_series " - + "WHERE NOT EXISTS (" - + " SELECT 1 FROM table_entity te " - + " WHERE te.fqnHash = profiler_data_time_series.entityFQNHash" - + ") " - + "LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM profiler_data_time_series " - + "WHERE ctid IN (" - + " SELECT pdts.ctid FROM profiler_data_time_series pdts " - + " WHERE NOT EXISTS (" - + " SELECT 1 FROM table_entity te " - + " WHERE te.fqnHash = pdts.entityFQNHash" - + " ) " - + " LIMIT :limit" - + ")", - connectionType = POSTGRES) - int deleteOrphanedRecords(@Bind("limit") int limit); - - record LatestExtensionRecord(String entityFQNHash, String json) {} - - class LatestExtensionRecordMapper implements RowMapper { - @Override - public LatestExtensionRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new LatestExtensionRecord(rs.getString("entityFQNHash"), rs.getString("json")); - } - } - } - - interface DataQualityDataTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "data_quality_data_time_series"; - } - - @RegisterRowMapper(LatestRecordWithFQNHashMapper.class) - @SqlQuery( - "SELECT t1.entityFQNHash, t1.json FROM data_quality_data_time_series t1 " - + "INNER JOIN (SELECT entityFQNHash, MAX(timestamp) as maxTs " - + "FROM data_quality_data_time_series WHERE entityFQNHash IN () " - + "GROUP BY entityFQNHash) t2 " - + "ON t1.entityFQNHash = t2.entityFQNHash AND t1.timestamp = t2.maxTs") - List getLatestRecordBatch( - @BindListFQN("entityFQNHashes") List entityFQNs); - - @SqlUpdate( - "DELETE FROM data_quality_data_time_series WHERE entityFQNHash = :testCaseFQNHash AND extension = 'testCase.testCaseResult'") - void deleteAll(@BindFQN("testCaseFQNHash") String entityFQNHash); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) " - + "VALUES (:testCaseFQNHash, :extension, :jsonSchema, :json, :incidentStateId)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) " - + "VALUES (:testCaseFQNHash, :extension, :jsonSchema, (:json :: jsonb), :incidentStateId)", - connectionType = POSTGRES) - void insert( - @Define("table") String table, - @BindFQN("testCaseFQNHash") String testCaseFQNHash, - @Bind("extension") String extension, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json, - @Bind("incidentStateId") String incidentStateId); - - default void insert( - String entityFQNHash, - String extension, - String jsonSchema, - String json, - String incidentStateId) { - insert(getTimeSeriesTableName(), entityFQNHash, extension, jsonSchema, json, incidentStateId); - } - } - - class LatestRecordWithFQNHash { - private final String entityFQNHash; - private final String json; - - public LatestRecordWithFQNHash(String entityFQNHash, String json) { - this.entityFQNHash = entityFQNHash; - this.json = json; - } - - public String getEntityFQNHash() { - return entityFQNHash; - } - - public String getJson() { - return json; - } - } - - class LatestRecordWithFQNHashMapper implements RowMapper { - @Override - public LatestRecordWithFQNHash map(ResultSet r, StatementContext ctx) throws SQLException { - return new LatestRecordWithFQNHash(r.getString("entityFQNHash"), r.getString("json")); - } - } - - interface QueryCostTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "query_cost_time_series"; - } - - // TODO: Do not change id on override... updating json changed the id as well - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO
(entityFQNHash, jsonSchema, json) " - + "VALUES (:entityFQNHash, :jsonSchema, :json) ON DUPLICATE KEY UPDATE" - + " json = VALUES(json);", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO
(entityFQNHash, jsonSchema, json) " - + "VALUES (:entityFQNHash, :jsonSchema, (:json :: jsonb)) " - + "ON CONFLICT (entityFQNHash, timestamp) " - + "DO UPDATE SET " - + "json = EXCLUDED.json", - connectionType = POSTGRES) - void insertWithoutExtension( - @Define("table") String table, - @BindFQN("entityFQNHash") String entityFQNHash, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json); - - @SqlUpdate("DELETE FROM query_cost_time_series WHERE entityFQNHash = :entityFQNHash ") - void deleteWithEntityFqnHash(@BindFQN("entityFQNHash") String entityFQNHash); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM query_cost_time_series " - + "WHERE id IN (" - + " SELECT id FROM (" - + " SELECT qcts.id FROM query_cost_time_series qcts " - + " LEFT JOIN query_entity qe ON qcts.entityFQNHash = qe.fqnHash " - + " WHERE qe.fqnHash IS NULL " - + " LIMIT :limit" - + " ) sub" - + ")", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM query_cost_time_series " - + "WHERE id IN (" - + " SELECT qcts.id FROM query_cost_time_series qcts " - + " LEFT JOIN query_entity qe ON qcts.entityFQNHash = qe.fqnHash " - + " WHERE qe.fqnHash IS NULL " - + " LIMIT :limit" - + ")", - connectionType = POSTGRES) - int deleteOrphanedRecords(@Bind("limit") int limit); - } - - interface TestCaseResolutionStatusTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "test_case_resolution_status_time_series"; - } - - @SqlQuery( - value = - "SELECT json FROM test_case_resolution_status_time_series " - + "WHERE stateId = :stateId ORDER BY timestamp DESC") - List listTestCaseResolutionStatusesForStateId(@Bind("stateId") String stateId); - - @SqlQuery( - value = - "SELECT json FROM test_case_resolution_status_time_series " - + "WHERE entityFQNHash = :entityFQNHash ORDER BY timestamp DESC") - List listTestCaseResolutionForEntityFQNHash( - @BindFQN("entityFQNHash") String entityFqnHas); - - @SqlQuery( - value = - "SELECT json FROM test_case_resolution_status_time_series " - + "WHERE assignee = :userFqn ORDER BY timestamp DESC") - List listTestCaseResolutionForAssignee(@Bind("userFqn") String userFqn); - - @SqlQuery( - value = - "SELECT json FROM test_case_resolution_status_time_series " - + "WHERE stateId = :stateId ORDER BY timestamp ASC LIMIT 1") - String listFirstTestCaseResolutionStatusesForStateId(@Bind("stateId") String stateId); - - @SqlUpdate( - "DELETE FROM test_case_resolution_status_time_series WHERE entityFQNHash = :entityFQNHash") - void delete(@BindFQN("entityFQNHash") String entityFQNHash); - - @SqlQuery( - "SELECT json FROM " - + "(SELECT id, json, testCaseResolutionStatusType, assignee, ROW_NUMBER() OVER(PARTITION BY ORDER BY timestamp DESC) AS row_num " - + "FROM
" - + "AND timestamp BETWEEN :startTs AND :endTs " - + "ORDER BY timestamp DESC) ranked " - + " AND ranked.row_num = 1 LIMIT :limit OFFSET :offset") - List listWithOffset( - @Define("table") String table, - @BindMap Map params, - @Define("cond") String cond, - @Define("partition") String partition, - @Bind("limit") int limit, - @Bind("offset") int offset, - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs, - @BindMap Map outerParams, - @Define("outerCond") String outerFilter); - - @Override - default List listWithOffset( - ListFilter filter, int limit, int offset, Long startTs, Long endTs, boolean latest) { - if (latest) { - // When fetching latest, we need to apply Assignee and Status filters on the outer query - // i.e. after we have fetched the latest records for each testCaseFQNHash - // We'll first get the values, remove then from `filter` and then create `outerFilter` - String testCaseResolutionStatusType = filter.getQueryParam("testCaseResolutionStatusType"); - filter.removeQueryParam("testCaseResolutionStatusType"); - String assignee = filter.getQueryParam("assignee"); - filter.removeQueryParam("assignee"); - - ListFilter outerFilter = new ListFilter(null); - outerFilter.addQueryParam("testCaseResolutionStatusType", testCaseResolutionStatusType); - outerFilter.addQueryParam("assignee", assignee); - - String condition = filter.getCondition(); - condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); - - return listWithOffset( - getTimeSeriesTableName(), - filter.getQueryParams(), - condition, - getPartitionFieldName(), - limit, - offset, - startTs, - endTs, - filter.getQueryParams(), - outerFilter.getCondition()); - } - String condition = filter.getCondition(); - condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); - return listWithOffset( - getTimeSeriesTableName(), - filter.getQueryParams(), - condition, - limit, - offset, - startTs, - endTs); - } - - @Override - default int listCount(ListFilter filter, Long startTs, Long endTs, boolean latest) { - String condition = filter.getCondition(); - condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); - return latest - ? listCount( - getTimeSeriesTableName(), - getPartitionFieldName(), - filter.getQueryParams(), - condition, - startTs, - endTs) - : listCount(getTimeSeriesTableName(), filter.getQueryParams(), condition, startTs, endTs); - } - - @Override - default List listWithOffset(ListFilter filter, int limit, int offset) { - String condition = filter.getCondition(); - condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); - return listWithOffset( - getTimeSeriesTableName(), filter.getQueryParams(), condition, limit, offset); - } - - @Override - default int listCount(ListFilter filter) { - String condition = filter.getCondition(); - condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); - return listCount(getTimeSeriesTableName(), filter.getQueryParams(), condition); - } - - // relation = 9 corresponds to Relationship.PARENT_OF (the enum ordinal is stable; - // see Relationship.java where new values must be appended). The annotation can't - // reference the enum at compile time, so we inline the ordinal here. - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM test_case_resolution_status_time_series " - + "WHERE id IN (" - + " SELECT id FROM (" - + " SELECT ts.id FROM test_case_resolution_status_time_series ts " - + " LEFT JOIN entity_relationship er " - + " ON er.toId = ts.id AND er.relation = 9 " // 9 = Relationship.PARENT_OF - + " AND er.fromEntity = 'testCase' " - + " AND er.toEntity = 'testCaseResolutionStatus' " - + " WHERE er.toId IS NULL " - + " LIMIT :limit" - + " ) sub" - + ")", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM test_case_resolution_status_time_series " - + "WHERE id IN (" - + " SELECT ts.id FROM test_case_resolution_status_time_series ts " - + " LEFT JOIN entity_relationship er " - + " ON er.toId = ts.id AND er.relation = 9 " // 9 = Relationship.PARENT_OF - + " AND er.fromEntity = 'testCase' " - + " AND er.toEntity = 'testCaseResolutionStatus' " - + " WHERE er.toId IS NULL " - + " LIMIT :limit" - + ")", - connectionType = POSTGRES) - int deleteOrphanedRecords(@Bind("limit") int limit); - } - - interface TestCaseResultTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "data_quality_data_time_series"; - } - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) " - + "VALUES (:testCaseFQNHash, :extension, :jsonSchema, :json, :incidentStateId)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) " - + "VALUES (:testCaseFQNHash, :extension, :jsonSchema, (:json :: jsonb), :incidentStateId)", - connectionType = POSTGRES) - void insert( - @Define("table") String table, - @BindFQN("testCaseFQNHash") String testCaseFQNHash, - @Bind("extension") String extension, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json, - @Bind("incidentStateId") String incidentStateId); - - @ConnectionAwareSqlQuery( - value = - """ - SELECT dqdts1.json FROM - data_quality_data_time_series dqdts1 - INNER JOIN ( - SELECT tc.fqnHash - FROM entity_relationship er - INNER JOIN test_case tc ON er.toId = tc.id - WHERE fromEntity = 'testSuite' AND toEntity = 'testCase' AND fromId = :testSuiteId - ) ts ON dqdts1.entityFQNHash = ts.fqnHash - LEFT JOIN data_quality_data_time_series dqdts2 FORCE INDEX (idx_entity_timestamp_desc) ON - (dqdts1.entityFQNHash = dqdts2.entityFQNHash AND dqdts1.timestamp < dqdts2.timestamp) - WHERE dqdts2.entityFQNHash IS NULL""", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - """ - SELECT dqdts1.json FROM - data_quality_data_time_series dqdts1 - INNER JOIN ( - SELECT tc.fqnHash - FROM entity_relationship er - INNER JOIN test_case tc ON er.toId = tc.id - WHERE fromEntity = 'testSuite' AND toEntity = 'testCase' AND fromId = :testSuiteId - ) ts ON dqdts1.entityFQNHash = ts.fqnHash - LEFT JOIN data_quality_data_time_series dqdts2 ON - (dqdts1.entityFQNHash = dqdts2.entityFQNHash AND dqdts1.timestamp < dqdts2.timestamp) - WHERE dqdts2.entityFQNHash IS NULL""", - connectionType = POSTGRES) - List listLastTestCaseResultsForTestSuite(@BindMap Map params); - - @SqlQuery( - """ - SELECT dqdts1.json FROM - data_quality_data_time_series dqdts1 - LEFT JOIN data_quality_data_time_series dqdts2 ON - (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp) - WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""") - String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN); - - default void insert( - String testCaseFQN, - String extension, - String jsonSchema, - String json, - UUID incidentStateId) { - - insert( - getTimeSeriesTableName(), - testCaseFQN, - extension, - jsonSchema, - json, - incidentStateId != null ? incidentStateId.toString() : null); - } - - default List listLastTestCaseResultsForTestSuite(UUID testSuiteId) { - return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString())); - } - - record ResultSummaryRow( - String testSuiteId, String testCaseFQN, String testCaseStatus, long timestamp) {} - - class ResultSummaryRowMapper implements RowMapper { - @Override - public ResultSummaryRow map(ResultSet rs, StatementContext ctx) throws SQLException { - return new ResultSummaryRow( - rs.getString("testSuiteId"), - rs.getString("testCaseFQN"), - rs.getString("testCaseStatus"), - rs.getLong("timestamp")); - } - } - - @ConnectionAwareSqlQuery( - value = - """ - WITH suite_test_cases AS ( - SELECT tc.fqnHash, er.fromId as testSuiteId - FROM entity_relationship er - INNER JOIN test_case tc ON er.toId = tc.id - WHERE er.fromEntity = 'testSuite' AND er.toEntity = 'testCase' - AND er.fromId IN () - ), - latest_results AS ( - SELECT dqdts.entityFQNHash, - JSON_UNQUOTE(JSON_EXTRACT(dqdts.json, '$.testCaseFQN')) as testCaseFQN, - JSON_UNQUOTE(JSON_EXTRACT(dqdts.json, '$.testCaseStatus')) as testCaseStatus, - dqdts.timestamp, - ROW_NUMBER() OVER (PARTITION BY dqdts.entityFQNHash ORDER BY dqdts.timestamp DESC) as rn - FROM data_quality_data_time_series dqdts - WHERE dqdts.entityFQNHash IN (SELECT fqnHash FROM suite_test_cases) - ) - SELECT stc.testSuiteId, lr.testCaseFQN, lr.testCaseStatus, lr.timestamp - FROM latest_results lr - INNER JOIN suite_test_cases stc ON lr.entityFQNHash = stc.fqnHash - WHERE lr.rn = 1 - """, - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - """ - WITH suite_test_cases AS ( - SELECT tc.fqnHash, er.fromId as testSuiteId - FROM entity_relationship er - INNER JOIN test_case tc ON er.toId = tc.id - WHERE er.fromEntity = 'testSuite' AND er.toEntity = 'testCase' - AND er.fromId IN () - ), - latest_results AS ( - SELECT dqdts.entityFQNHash, - dqdts.json->>'testCaseFQN' as testCaseFQN, - dqdts.json->>'testCaseStatus' as testCaseStatus, - dqdts.timestamp, - ROW_NUMBER() OVER (PARTITION BY dqdts.entityFQNHash ORDER BY dqdts.timestamp DESC) as rn - FROM data_quality_data_time_series dqdts - WHERE dqdts.entityFQNHash IN (SELECT fqnHash FROM suite_test_cases) - ) - SELECT stc.testSuiteId, lr.testCaseFQN, lr.testCaseStatus, lr.timestamp - FROM latest_results lr - INNER JOIN suite_test_cases stc ON lr.entityFQNHash = stc.fqnHash - WHERE lr.rn = 1 - """, - connectionType = POSTGRES) - @UseRowMapper(ResultSummaryRowMapper.class) - List listResultSummariesForTestSuites( - @BindList("testSuiteIds") List testSuiteIds); - - record SuiteMaxTimestamp(String testSuiteId, long maxTimestamp) {} - - class SuiteMaxTimestampMapper implements RowMapper { - @Override - public SuiteMaxTimestamp map(ResultSet rs, StatementContext ctx) throws SQLException { - return new SuiteMaxTimestamp(rs.getString("testSuiteId"), rs.getLong("maxTimestamp")); - } - } - - @SqlQuery( - """ - SELECT er_sub.fromId as testSuiteId, MAX(dqdts.timestamp) as maxTimestamp - FROM data_quality_data_time_series dqdts - INNER JOIN ( - SELECT tc.fqnHash, er.fromId - FROM entity_relationship er - INNER JOIN test_case tc ON er.toId = tc.id - WHERE er.fromEntity = 'testSuite' AND er.toEntity = 'testCase' - AND er.fromId IN () - ) er_sub ON dqdts.entityFQNHash = er_sub.fqnHash - GROUP BY er_sub.fromId""") - @UseRowMapper(SuiteMaxTimestampMapper.class) - List getMaxTimestampForTestSuites( - @BindList("testSuiteIds") List testSuiteIds); - } - - interface TestCaseDimensionResultTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "test_case_dimension_results_time_series"; - } - - @SqlQuery( - "SELECT json FROM test_case_dimension_results_time_series " - + "WHERE entityFQNHash = :testCaseFQN AND timestamp >= :startTs AND timestamp <= :endTs " - + "ORDER BY timestamp DESC") - List listTestCaseDimensionResults( - @BindFQN("testCaseFQN") String testCaseFQN, - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs); - - @SqlQuery( - "SELECT json FROM test_case_dimension_results_time_series " - + "WHERE entityFQNHash = :testCaseFQN AND dimensionKey = :dimensionKey AND timestamp >= :startTs AND timestamp <= :endTs " - + "ORDER BY timestamp DESC") - List listTestCaseDimensionResultsByKey( - @BindFQN("testCaseFQN") String testCaseFQN, - @Bind("dimensionKey") String dimensionKey, - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs); - - @SqlQuery( - "SELECT json FROM test_case_dimension_results_time_series " - + "WHERE entityFQNHash = :testCaseFQN AND dimensionName = :dimensionName AND timestamp >= :startTs AND timestamp <= :endTs " - + "ORDER BY timestamp DESC") - List listTestCaseDimensionResultsByDimensionName( - @BindFQN("testCaseFQN") String testCaseFQN, - @Bind("dimensionName") String dimensionName, - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs); - - @SqlQuery( - "SELECT DISTINCT dimensionKey FROM test_case_dimension_results_time_series " - + "WHERE entityFQNHash = :testCaseFQN AND timestamp >= :startTs AND timestamp <= :endTs") - List listAvailableDimensionKeys( - @BindFQN("testCaseFQN") String testCaseFQN, - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs); - - @SqlUpdate( - "DELETE FROM test_case_dimension_results_time_series WHERE entityFQNHash = :testCaseFQNHash") - void deleteAll(@BindFQN("testCaseFQNHash") String testCaseFQN); - } - - class EntitiesCountRowMapper implements RowMapper { - @Override - public EntitiesCount map(ResultSet rs, StatementContext ctx) throws SQLException { - return new EntitiesCount() - .withTableCount(rs.getInt("tableCount")) - .withTopicCount(rs.getInt("topicCount")) - .withDashboardCount(rs.getInt("dashboardCount")) - .withPipelineCount(rs.getInt("pipelineCount")) - .withMlmodelCount(rs.getInt("mlmodelCount")) - .withServicesCount(rs.getInt("servicesCount")) - .withUserCount(rs.getInt("userCount")) - .withTeamCount(rs.getInt("teamCount")) - .withTestSuiteCount(rs.getInt("testSuiteCount")) - .withStorageContainerCount(rs.getInt("storageContainerCount")) - .withGlossaryCount(rs.getInt("glossaryCount")) - .withGlossaryTermCount(rs.getInt("glossaryTermCount")); - } - } - - class ServicesCountRowMapper implements RowMapper { - @Override - public ServicesCount map(ResultSet rs, StatementContext ctx) throws SQLException { - return new ServicesCount() - .withDatabaseServiceCount(rs.getInt("databaseServiceCount")) - .withMessagingServiceCount(rs.getInt("messagingServiceCount")) - .withDashboardServiceCount(rs.getInt("dashboardServiceCount")) - .withPipelineServiceCount(rs.getInt("pipelineServiceCount")) - .withMlModelServiceCount(rs.getInt("mlModelServiceCount")) - .withStorageServiceCount(rs.getInt("storageServiceCount")); - } - } - - interface SystemDAO { - @ConnectionAwareSqlQuery( - value = - "SELECT (SELECT COUNT(fqnHash) FROM table_entity ) as tableCount, " - + "(SELECT COUNT(fqnHash) FROM topic_entity ) as topicCount, " - + "(SELECT COUNT(fqnHash) FROM dashboard_entity ) as dashboardCount, " - + "(SELECT COUNT(fqnHash) FROM pipeline_entity ) as pipelineCount, " - + "(SELECT COUNT(fqnHash) FROM ml_model_entity ) as mlmodelCount, " - + "(SELECT COUNT(fqnHash) FROM storage_container_entity ) as storageContainerCount, " - + "(SELECT COUNT(fqnHash) FROM search_index_entity ) as searchIndexCount, " - + "(SELECT COUNT(nameHash) FROM glossary_entity ) as glossaryCount, " - + "(SELECT COUNT(fqnHash) FROM glossary_term_entity ) as glossaryTermCount, " - + "(SELECT (SELECT COUNT(nameHash) FROM metadata_service_entity ) + " - + "(SELECT COUNT(nameHash) FROM dbservice_entity )+" - + "(SELECT COUNT(nameHash) FROM messaging_service_entity )+ " - + "(SELECT COUNT(nameHash) FROM dashboard_service_entity )+ " - + "(SELECT COUNT(nameHash) FROM pipeline_service_entity )+ " - + "(SELECT COUNT(nameHash) FROM mlmodel_service_entity )+ " - + "(SELECT COUNT(nameHash) FROM search_service_entity )+ " - + "(SELECT COUNT(nameHash) FROM storage_service_entity )) as servicesCount, " - + "(SELECT COUNT(nameHash) FROM user_entity AND (JSON_EXTRACT(json, '$.isBot') IS NULL OR JSON_EXTRACT(json, '$.isBot') = FALSE)) as userCount, " - + "(SELECT COUNT(nameHash) FROM team_entity ) as teamCount, " - + "(SELECT COUNT(fqnHash) FROM test_suite ) as testSuiteCount", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT (SELECT COUNT(*) FROM table_entity ) as tableCount, " - + "(SELECT COUNT(*) FROM topic_entity ) as topicCount, " - + "(SELECT COUNT(*) FROM dashboard_entity ) as dashboardCount, " - + "(SELECT COUNT(*) FROM pipeline_entity ) as pipelineCount, " - + "(SELECT COUNT(*) FROM ml_model_entity ) as mlmodelCount, " - + "(SELECT COUNT(*) FROM storage_container_entity ) as storageContainerCount, " - + "(SELECT COUNT(*) FROM search_index_entity ) as searchIndexCount, " - + "(SELECT COUNT(*) FROM glossary_entity ) as glossaryCount, " - + "(SELECT COUNT(*) FROM glossary_term_entity ) as glossaryTermCount, " - + "(SELECT (SELECT COUNT(*) FROM metadata_service_entity ) + " - + "(SELECT COUNT(*) FROM dbservice_entity )+" - + "(SELECT COUNT(*) FROM messaging_service_entity )+ " - + "(SELECT COUNT(*) FROM dashboard_service_entity )+ " - + "(SELECT COUNT(*) FROM pipeline_service_entity )+ " - + "(SELECT COUNT(*) FROM mlmodel_service_entity )+ " - + "(SELECT COUNT(*) FROM search_service_entity )+ " - + "(SELECT COUNT(*) FROM storage_service_entity )) as servicesCount, " - + "(SELECT COUNT(*) FROM user_entity AND (json#>'{isBot}' IS NULL OR ((json#>'{isBot}')::boolean) = FALSE)) as userCount, " - + "(SELECT COUNT(*) FROM team_entity ) as teamCount, " - + "(SELECT COUNT(*) FROM test_suite ) as testSuiteCount", - connectionType = POSTGRES) - @RegisterRowMapper(EntitiesCountRowMapper.class) - EntitiesCount getAggregatedEntitiesCount(@Define("cond") String cond) throws StatementException; - - @ConnectionAwareSqlQuery( - value = - "SELECT (SELECT COUNT(nameHash) FROM dbservice_entity ) as databaseServiceCount, " - + "(SELECT COUNT(nameHash) FROM messaging_service_entity ) as messagingServiceCount, " - + "(SELECT COUNT(nameHash) FROM dashboard_service_entity ) as dashboardServiceCount, " - + "(SELECT COUNT(nameHash) FROM pipeline_service_entity ) as pipelineServiceCount, " - + "(SELECT COUNT(nameHash) FROM mlmodel_service_entity ) as mlModelServiceCount, " - + "(SELECT COUNT(nameHash) FROM storage_service_entity ) as storageServiceCount, " - + "(SELECT COUNT(nameHash) FROM search_service_entity ) as searchServiceCount", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT (SELECT COUNT(*) FROM dbservice_entity ) as databaseServiceCount, " - + "(SELECT COUNT(*) FROM messaging_service_entity ) as messagingServiceCount, " - + "(SELECT COUNT(*) FROM dashboard_service_entity ) as dashboardServiceCount, " - + "(SELECT COUNT(*) FROM pipeline_service_entity ) as pipelineServiceCount, " - + "(SELECT COUNT(*) FROM mlmodel_service_entity ) as mlModelServiceCount, " - + "(SELECT COUNT(*) FROM storage_service_entity ) as storageServiceCount, " - + "(SELECT COUNT(*) FROM search_service_entity ) as searchServiceCount", - connectionType = POSTGRES) - @RegisterRowMapper(ServicesCountRowMapper.class) - ServicesCount getAggregatedServicesCount(@Define("cond") String cond) throws StatementException; - - @SqlQuery("SELECT configType,json FROM openmetadata_settings") - @RegisterRowMapper(SettingsRowMapper.class) - List getAllConfig() throws StatementException; - - @SqlQuery("SELECT configType, json FROM openmetadata_settings WHERE configType = :configType") - @RegisterRowMapper(SettingsRowMapper.class) - Settings getConfigWithKey(@Bind("configType") String configType) throws StatementException; - - @ConnectionAwareSqlUpdate( - value = - "INSERT into openmetadata_settings (configType, json)" - + "VALUES (:configType, :json) ON DUPLICATE KEY UPDATE json = :json", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT into openmetadata_settings (configType, json)" - + "VALUES (:configType, :json :: jsonb) ON CONFLICT (configType) DO UPDATE SET json = EXCLUDED.json", - connectionType = POSTGRES) - void insertSettings(@Bind("configType") String configType, @Bind("json") String json); - - @SqlUpdate(value = "DELETE from openmetadata_settings WHERE configType = :configType") - void delete(@Bind("configType") String configType); - - @SqlQuery("SELECT 42") - Integer testConnection() throws StatementException; - - @ConnectionAwareSqlQuery( - value = - "SELECT JSON_EXTRACT(json, '$.fullyQualifiedName') FROM
WHERE id NOT IN ( SELECT toId FROM entity_relationship WHERE fromEntity = :fromEntity AND toEntity = :toEntity)", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json ->> 'fullyQualifiedName' FROM
WHERE id NOT IN ( SELECT toId FROM entity_relationship WHERE fromEntity = :fromEntity AND toEntity = :toEntity)", - connectionType = POSTGRES) - List getBrokenRelationFromParentToChild( - @Define("table") String tableName, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity); - - @SqlUpdate( - value = - "DELETE FROM
WHERE id NOT IN (SELECT toId FROM entity_relationship WHERE fromEntity = :fromEntity AND toEntity = :toEntity)") - int deleteBrokenRelationFromParentToChild( - @Define("table") String tableName, - @Bind("fromEntity") String fromEntity, - @Bind("toEntity") String toEntity); - } - - class SettingsRowMapper implements RowMapper { - @Override - public Settings map(ResultSet rs, StatementContext ctx) throws SQLException { - return getSettings(SettingsType.fromValue(rs.getString("configType")), rs.getString("json")); - } - - public static Settings getSettings(SettingsType configType, String json) { - Settings settings = new Settings(); - settings.setConfigType(configType); - Object value = - switch (configType) { - case EMAIL_CONFIGURATION -> JsonUtils.readValue(json, SmtpSettings.class); - case OPEN_METADATA_BASE_URL_CONFIGURATION -> JsonUtils.readValue( - json, OpenMetadataBaseUrlConfiguration.class); - case CUSTOM_UI_THEME_PREFERENCE -> JsonUtils.readValue(json, UiThemePreference.class); - case LOGIN_CONFIGURATION -> JsonUtils.readValue(json, LoginConfiguration.class); - case SLACK_APP_CONFIGURATION, SLACK_INSTALLER, SLACK_BOT, SLACK_STATE -> JsonUtils - .readValue(json, String.class); - case PROFILER_CONFIGURATION -> JsonUtils.readValue(json, ProfilerConfiguration.class); - case SEARCH_SETTINGS -> JsonUtils.readValue(json, SearchSettings.class); - case ASSET_CERTIFICATION_SETTINGS -> JsonUtils.readValue( - json, AssetCertificationSettings.class); - case WORKFLOW_SETTINGS -> JsonUtils.readValue(json, WorkflowSettings.class); - case LINEAGE_SETTINGS -> JsonUtils.readValue(json, LineageSettings.class); - case AUTHENTICATION_CONFIGURATION -> JsonUtils.readValue( - json, AuthenticationConfiguration.class); - case AUTHORIZER_CONFIGURATION -> JsonUtils.readValue( - json, AuthorizerConfiguration.class); - case ENTITY_RULES_SETTINGS -> JsonUtils.readValue(json, EntityRulesSettings.class); - case SCIM_CONFIGURATION -> JsonUtils.readValue(json, ScimConfiguration.class); - case OPEN_LINEAGE_SETTINGS -> JsonUtils.readValue(json, OpenLineageSettings.class); - case TEAMS_APP_CONFIGURATION -> JsonUtils.readValue(json, TeamsAppConfiguration.class); - case MCP_CONFIGURATION -> JsonUtils.readValue(json, MCPConfiguration.class); - case GLOSSARY_TERM_RELATION_SETTINGS -> JsonUtils.readValue( - json, GlossaryTermRelationSettings.class); - default -> throw new IllegalArgumentException("Invalid Settings Type " + configType); - }; - settings.setConfigValue(value); - return settings; - } - } - - class TokenRowMapper implements RowMapper { - @Override - public TokenInterface map(ResultSet rs, StatementContext ctx) throws SQLException { - return getToken(TokenType.fromValue(rs.getString("tokenType")), rs.getString("json")); - } - - public static TokenInterface getToken(TokenType type, String json) { - return switch (type) { - case EMAIL_VERIFICATION -> JsonUtils.readValue(json, EmailVerificationToken.class); - case PASSWORD_RESET -> JsonUtils.readValue(json, PasswordResetToken.class); - case REFRESH_TOKEN -> JsonUtils.readValue(json, RefreshToken.class); - case PERSONAL_ACCESS_TOKEN -> JsonUtils.readValue(json, PersonalAccessToken.class); - case SUPPORT_TOKEN -> JsonUtils.readValue(json, SupportToken.class); - }; - } - } - - class UserSessionRowMapper implements RowMapper { - @Override - public UserSession map(ResultSet rs, StatementContext ctx) throws SQLException { - return JsonUtils.readValue(rs.getString("json"), UserSession.class); - } - } - - // OAuth 2.0 Row Mappers - class OAuthClientRowMapper implements RowMapper { - @Override - public OAuthRecords.OAuthClientRecord map(ResultSet rs, StatementContext ctx) - throws SQLException { - return new OAuthRecords.OAuthClientRecord( - UUID.fromString(rs.getString("id")), - rs.getString("client_id"), - rs.getString("client_secret_encrypted"), - rs.getString("client_name"), - JsonUtils.readObjects(rs.getString("redirect_uris"), String.class), - JsonUtils.readObjects(rs.getString("grant_types"), String.class), - rs.getString("token_endpoint_auth_method"), - JsonUtils.readObjects(rs.getString("scopes"), String.class)); - } - } - - class OAuthAuthorizationCodeRowMapper - implements RowMapper { - @Override - public OAuthRecords.OAuthAuthorizationCodeRecord map(ResultSet rs, StatementContext ctx) - throws SQLException { - return new OAuthRecords.OAuthAuthorizationCodeRecord( - rs.getString("code"), - rs.getString("client_id"), - rs.getString("user_name"), - rs.getString("code_challenge"), - rs.getString("code_challenge_method"), - rs.getString("redirect_uri"), - JsonUtils.readObjects(rs.getString("scopes"), String.class), - rs.getLong("expires_at"), - rs.getBoolean("used")); - } - } - - class OAuthAccessTokenRowMapper implements RowMapper { - @Override - public OAuthRecords.OAuthAccessTokenRecord map(ResultSet rs, StatementContext ctx) - throws SQLException { - return new OAuthRecords.OAuthAccessTokenRecord( - UUID.fromString(rs.getString("id")), - rs.getString("token_hash"), - rs.getString("access_token_encrypted"), - rs.getString("client_id"), - rs.getString("user_name"), - JsonUtils.readObjects(rs.getString("scopes"), String.class), - rs.getLong("expires_at")); - } - } - - class OAuthRefreshTokenRowMapper implements RowMapper { - @Override - public OAuthRecords.OAuthRefreshTokenRecord map(ResultSet rs, StatementContext ctx) - throws SQLException { - return new OAuthRecords.OAuthRefreshTokenRecord( - UUID.fromString(rs.getString("id")), - rs.getString("token_hash"), - rs.getString("refresh_token_encrypted"), - rs.getString("client_id"), - rs.getString("user_name"), - JsonUtils.readObjects(rs.getString("scopes"), String.class), - rs.getLong("expires_at"), - rs.getBoolean("revoked")); - } - } - - interface TokenDAO { - @SqlQuery("SELECT tokenType, json FROM user_tokens WHERE token = :token") - @RegisterRowMapper(TokenRowMapper.class) - TokenInterface findByToken(@Bind("token") String token) throws StatementException; - - @SqlQuery( - "SELECT tokenType, json FROM user_tokens WHERE userId = :userId AND tokenType = :tokenType ") - @RegisterRowMapper(TokenRowMapper.class) - List getAllUserTokenWithType( - @BindUUID("userId") UUID userId, @Bind("tokenType") String tokenType) - throws StatementException; - - @ConnectionAwareSqlUpdate( - value = "INSERT INTO user_tokens (json) VALUES (:json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "INSERT INTO user_tokens (json) VALUES (:json :: jsonb)", - connectionType = POSTGRES) - void insert(@Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = "UPDATE user_tokens SET json = :json WHERE token = :token", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE user_tokens SET json = (:json :: jsonb) WHERE token = :token", - connectionType = POSTGRES) - void update(@Bind("token") String token, @Bind("json") String json); - - @SqlUpdate(value = "DELETE from user_tokens WHERE token = :token") - void delete(@Bind("token") String token); - - @SqlUpdate(value = "DELETE from user_tokens WHERE token IN ()") - void deleteAll(@BindList("tokenIds") List tokens); - - @SqlUpdate(value = "DELETE from user_tokens WHERE userid = :userid AND tokenType = :tokenType") - void deleteTokenByUserAndType( - @BindUUID("userid") UUID userid, @Bind("tokenType") String tokenType); - } - - interface UserSessionDAO { - @SqlQuery("SELECT json FROM user_session WHERE id = :id") - @RegisterRowMapper(UserSessionRowMapper.class) - UserSession findById(@Bind("id") String id) throws StatementException; - - @SqlQuery( - "SELECT json FROM user_session WHERE userId = :userId AND status = :status " - + "ORDER BY COALESCE(lastAccessedAt, 0) ASC LIMIT :limit") - @RegisterRowMapper(UserSessionRowMapper.class) - List findByUserIdAndStatus( - @Bind("userId") String userId, @Bind("status") String status, @Bind("limit") int limit) - throws StatementException; - - @SqlQuery( - "SELECT json FROM user_session " - + "WHERE status IN () AND expiresAt <= :now " - + "ORDER BY updatedAt ASC LIMIT :limit") - @RegisterRowMapper(UserSessionRowMapper.class) - List findSessionsExpiredByAbsoluteTimeout( - @BindList("statuses") List statuses, - @Bind("now") long now, - @Bind("limit") int limit) - throws StatementException; - - @SqlQuery( - "SELECT json FROM user_session " - + "WHERE status IN () AND idleExpiresAt <= :now " - + "ORDER BY updatedAt ASC LIMIT :limit") - @RegisterRowMapper(UserSessionRowMapper.class) - List findSessionsExpiredByIdleTimeout( - @BindList("statuses") List statuses, - @Bind("now") long now, - @Bind("limit") int limit) - throws StatementException; - - @SqlQuery( - "SELECT json FROM user_session WHERE status IN () AND updatedAt <= :cutoff " - + "ORDER BY updatedAt ASC LIMIT :limit") - @RegisterRowMapper(UserSessionRowMapper.class) - List findSessionsToPrune( - @BindList("statuses") List statuses, - @Bind("cutoff") long cutoff, - @Bind("limit") int limit) - throws StatementException; - - @ConnectionAwareSqlUpdate( - value = "INSERT INTO user_session (json) VALUES (:json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "INSERT INTO user_session (json) VALUES (:json :: jsonb)", - connectionType = POSTGRES) - void insert(@Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE user_session SET json = :json WHERE id = :id AND version = :expectedVersion", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE user_session SET json = (:json :: jsonb) WHERE id = :id AND version = :expectedVersion", - connectionType = POSTGRES) - int updateIfVersion( - @Bind("id") String id, - @Bind("expectedVersion") long expectedVersion, - @Bind("json") String json); - - @SqlUpdate("DELETE FROM user_session WHERE id = :id") - void delete(@Bind("id") String id); - - @SqlUpdate("DELETE FROM user_session WHERE id IN ()") - int deleteByIds(@BindList("ids") List ids); - } - - interface KpiDAO extends EntityDAO { - @Override - default String getTableName() { - return "kpi_entity"; - } - - @Override - default Class getEntityClass() { - return Kpi.class; - } - } - - interface WorkflowDAO extends EntityDAO { - @Override - default String getTableName() { - return "automations_workflow"; - } - - @Override - default Class getEntityClass() { - return Workflow.class; - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String workflowType = filter.getQueryParam("workflowType"); - String workflowStatus = filter.getQueryParam("workflowStatus"); - String condition = filter.getCondition(); - - if (workflowType == null && workflowStatus == null) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - - StringBuilder sqlCondition = new StringBuilder(); - sqlCondition.append(String.format("%s ", condition)); - - if (workflowType != null) { - sqlCondition.append("AND workflowType=:workflowType "); - } - - if (workflowStatus != null) { - sqlCondition.append("AND status=:workflowStatus "); - } - - return listBefore( - getTableName(), - filter.getQueryParams(), - sqlCondition.toString(), - limit, - beforeName, - beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String workflowType = filter.getQueryParam("workflowType"); - String workflowStatus = filter.getQueryParam("workflowStatus"); - String condition = filter.getCondition(); - - if (workflowType == null && workflowStatus == null) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - - StringBuilder sqlCondition = new StringBuilder(); - sqlCondition.append(String.format("%s ", condition)); - - if (workflowType != null) { - sqlCondition.append("AND workflowType=:workflowType "); - } - - if (workflowStatus != null) { - sqlCondition.append("AND status=:workflowStatus "); - } - - return listAfter( - getTableName(), - filter.getQueryParams(), - sqlCondition.toString(), - limit, - afterName, - afterId); - } - - @Override - default int listCount(ListFilter filter) { - String workflowType = filter.getQueryParam("workflowType"); - String workflowStatus = filter.getQueryParam("workflowStatus"); - String condition = filter.getCondition(); - - if (workflowType == null && workflowStatus == null) { - return EntityDAO.super.listCount(filter); - } - - StringBuilder sqlCondition = new StringBuilder(); - sqlCondition.append(String.format("%s ", condition)); - - if (workflowType != null) { - sqlCondition.append("AND workflowType=:workflowType "); - } - - if (workflowStatus != null) { - sqlCondition.append("AND status=:workflowStatus "); - } - - return listCount(getTableName(), filter.getQueryParams(), sqlCondition.toString()); - } - - @SqlQuery( - value = - "SELECT json FROM (" - + "SELECT name, id, json FROM
AND " - + "(
.name < :beforeName OR (
.name = :beforeName AND
.id < :beforeId)) " - + "ORDER BY name DESC,id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id") - List listBefore( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId); - - @SqlQuery( - value = - "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit") - List listAfter( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId); - - @SqlQuery(value = "SELECT count(*) FROM
") - int listCount( - @Define("table") String table, - @BindMap Map params, - @Define("sqlCondition") String sqlCondition); - } - - interface DataModelDAO extends EntityDAO { - @Override - default String getTableName() { - return "dashboard_data_model_entity"; - } - - @Override - default Class getEntityClass() { - return DashboardDataModel.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface DocStoreDAO extends EntityDAO { - @Override - default String getTableName() { - return "doc_store"; - } - - @Override - default Class getEntityClass() { - return Document.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - - @Override - default boolean supportsSoftDelete() { - return false; - } - - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String entityType = filter.getQueryParam("entityType"); - String fqnPrefix = filter.getQueryParam("fqnPrefix"); - String cond = filter.getCondition(); - if (entityType == null && fqnPrefix == null) { - return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); - } - - StringBuilder mysqlCondition = new StringBuilder(); - StringBuilder psqlCondition = new StringBuilder(); - mysqlCondition.append(cond); - psqlCondition.append(cond); - - if (fqnPrefix != null) { - String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); - filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); - filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); - String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; - mysqlCondition.append(fqnCond); - psqlCondition.append(fqnCond); - } - - if (entityType != null) { - mysqlCondition.append(" AND entityType=:entityType "); - psqlCondition.append(" AND entityType=:entityType "); - } - - return listBefore( - getTableName(), - filter.getQueryParams(), - mysqlCondition.toString(), - psqlCondition.toString(), - limit, - beforeName, - beforeId); - } - - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String entityType = filter.getQueryParam("entityType"); - String fqnPrefix = filter.getQueryParam("fqnPrefix"); - String cond = filter.getCondition(); - - if (entityType == null && fqnPrefix == null) { - return EntityDAO.super.listAfter(filter, limit, afterName, afterId); - } - - StringBuilder mysqlCondition = new StringBuilder(); - StringBuilder psqlCondition = new StringBuilder(); - mysqlCondition.append(cond); - psqlCondition.append(cond); - - if (fqnPrefix != null) { - String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); - filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); - filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); - String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; - mysqlCondition.append(fqnCond); - psqlCondition.append(fqnCond); - } - if (entityType != null) { - mysqlCondition.append(" AND entityType=:entityType "); - psqlCondition.append(" AND entityType=:entityType "); - } - - return listAfter( - getTableName(), - filter.getQueryParams(), - mysqlCondition.toString(), - psqlCondition.toString(), - limit, - afterName, - afterId); - } - - @Override - default int listCount(ListFilter filter) { - String entityType = filter.getQueryParam("entityType"); - String fqnPrefix = filter.getQueryParam("fqnPrefix"); - String cond = filter.getCondition(); - - if (entityType == null && fqnPrefix == null) { - return EntityDAO.super.listCount(filter); - } - - StringBuilder mysqlCondition = new StringBuilder(); - StringBuilder psqlCondition = new StringBuilder(); - mysqlCondition.append(cond); - psqlCondition.append(cond); - - if (fqnPrefix != null) { - String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); - filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); - filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); - String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; - mysqlCondition.append(fqnCond); - psqlCondition.append(fqnCond); - } - - if (entityType != null) { - mysqlCondition.append(" AND entityType=:entityType "); - psqlCondition.append(" AND entityType=:entityType "); - } - - return listCount( - getTableName(), - getNameHashColumn(), - filter.getQueryParams(), - mysqlCondition.toString(), - psqlCondition.toString()); - } - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT name, id, json FROM
AND " - + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " - + "ORDER BY name DESC,id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT name, id, json FROM
AND " - + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " - + "ORDER BY name DESC,id DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY name,id", - connectionType = POSTGRES) - List listBefore( - @Define("table") String table, - @BindMap Map params, - @Define("mysqlCond") String mysqlCond, - @Define("psqlCond") String psqlCond, - @Bind("limit") int limit, - @Bind("beforeName") String beforeName, - @Bind("beforeId") String beforeId); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit", - connectionType = POSTGRES) - List listAfter( - @Define("table") String table, - @BindMap Map params, - @Define("mysqlCond") String mysqlCond, - @Define("psqlCond") String psqlCond, - @Bind("limit") int limit, - @Bind("afterName") String afterName, - @Bind("afterId") String afterId); - - @ConnectionAwareSqlQuery( - value = "SELECT json FROM doc_store WHERE name = :name AND entityType = 'EmailTemplate'", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT json FROM doc_store WHERE name = :name AND entityType = 'EmailTemplate'", - connectionType = POSTGRES) - String fetchEmailTemplateByName(@Bind("name") String name); - - @ConnectionAwareSqlQuery( - value = "SELECT json FROM doc_store WHERE entityType = 'EmailTemplate'", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT json FROM doc_store WHERE entityType = 'EmailTemplate'", - connectionType = POSTGRES) - List fetchAllEmailTemplates(); - - @ConnectionAwareSqlUpdate( - value = "DELETE FROM doc_store WHERE entityType = 'EmailTemplate'", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "DELETE FROM doc_store WHERE entityType = 'EmailTemplate'", - connectionType = POSTGRES) - void deleteEmailTemplates(); - } - - interface LearningResourceDAO extends EntityDAO { - @Override - default String getTableName() { - return "learning_resource_entity"; - } - - @Override - default Class getEntityClass() { - return LearningResource.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface ContextMemoryDAO extends EntityDAO { - @Override - default String getTableName() { - return "context_memory"; - } - - @Override - default Class getEntityClass() { - return ContextMemory.class; - } - - @Override - default String getNameHashColumn() { - return "nameHash"; - } - } - - interface SuggestionDAO { - default String getTableName() { - return "suggestions"; - } - - @ConnectionAwareSqlUpdate( - value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json :: jsonb)", - connectionType = POSTGRES) - void insert(@BindFQN("fqnHash") String fullyQualifiedName, @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = "UPDATE suggestions SET json = :json where id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE suggestions SET json = (:json :: jsonb) where id = :id", - connectionType = POSTGRES) - void update(@BindUUID("id") UUID id, @Bind("json") String json); - - @SqlQuery("SELECT json FROM suggestions WHERE id = :id") - String findById(@BindUUID("id") UUID id); - - @SqlUpdate("DELETE FROM suggestions WHERE id = :id") - void delete(@BindUUID("id") UUID id); - - @SqlUpdate("DELETE FROM suggestions WHERE fqnHash = :fqnHash") - void deleteByFQN(@BindUUID("fqnHash") String fullyQualifiedName); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM suggestions suggestions WHERE JSON_EXTRACT(json, '$.createdBy.id') = :createdBy", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "DELETE FROM suggestions suggestions WHERE json #>> '{createdBy,id}' = :createdBy", - connectionType = POSTGRES) - void deleteByCreatedBy(@BindUUID("createdBy") UUID id); - - @SqlQuery("SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit") - List list( - @Bind("limit") int limit, - @Define("condition") String condition, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = "SELECT count(*) FROM suggestions ", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT count(*) FROM suggestions ", - connectionType = POSTGRES) - int listCount( - @Define("mysqlCond") String mysqlCond, - @Define("postgresCond") String postgresCond, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT updatedAt, json FROM suggestions " - + "ORDER BY updatedAt DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY updatedAt", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM (" - + "SELECT updatedAt, json FROM suggestions " - + "ORDER BY updatedAt DESC " - + "LIMIT :limit" - + ") last_rows_subquery ORDER BY updatedAt", - connectionType = POSTGRES) - List listBefore( - @Define("mysqlCond") String mysqlCond, - @Define("psqlCond") String psqlCond, - @Bind("limit") int limit, - @Bind("before") String before, - @BindMap Map params); - - @ConnectionAwareSqlQuery( - value = "SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit", - connectionType = POSTGRES) - List listAfter( - @Define("mysqlCond") String mysqlCond, - @Define("psqlCond") String psqlCond, - @Bind("limit") int limit, - @Bind("after") String after, - @BindMap Map params); - } - - interface APICollectionDAO extends EntityDAO { - @Override - default String getTableName() { - return "api_collection_entity"; - } - - @Override - default Class getEntityClass() { - return APICollection.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface APIEndpointDAO extends EntityDAO { - @Override - default String getTableName() { - return "api_endpoint_entity"; - } - - @Override - default Class getEntityClass() { - return APIEndpoint.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface WorkflowDefinitionDAO extends EntityDAO { - @Override - default String getTableName() { - return "workflow_definition_entity"; - } - - @Override - default Class getEntityClass() { - return WorkflowDefinition.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface WorkflowInstanceTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "workflow_instance_time_series"; - } - } - - interface WorkflowInstanceStateTimeSeriesDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "workflow_instance_state_time_series"; - } - - @SqlQuery( - value = - "SELECT json FROM workflow_instance_state_time_series " - + "WHERE workflowInstanceId = :workflowInstanceId AND stage = :stage ORDER BY timestamp DESC") - List listWorkflowInstanceStateForStage( - @Bind("workflowInstanceId") String workflowInstanceId, @Bind("stage") String stage); - - @SqlQuery( - value = - "SELECT json FROM workflow_instance_state_time_series " - + "WHERE workflowInstanceId = :workflowInstanceId ORDER BY timestamp ASC") - List listAllStatesForInstance(@Bind("workflowInstanceId") String workflowInstanceId); - } - - interface RecognizerFeedbackDAO { - @ConnectionAwareSqlUpdate( - value = "INSERT INTO recognizer_feedback_entity(json) VALUES (:json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "INSERT INTO recognizer_feedback_entity(json) VALUES (:json :: jsonb)", - connectionType = POSTGRES) - void insert(@Bind("json") String json); - - @SqlQuery("SELECT json FROM recognizer_feedback_entity WHERE id = :id") - String findById(@BindUUID("id") UUID id); - - @ConnectionAwareSqlUpdate( - value = "UPDATE recognizer_feedback_entity SET json = :json WHERE id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE recognizer_feedback_entity SET json = :json :: jsonb WHERE id = :id", - connectionType = POSTGRES) - void update(@BindUUID("id") UUID id, @Bind("json") String json); - - @SqlQuery("SELECT json FROM recognizer_feedback_entity WHERE entityLink = :entityLink") - List findByEntityLink(@Bind("entityLink") String entityLink); - - @SqlQuery("SELECT json FROM recognizer_feedback_entity WHERE tagFQN = :tagFQN") - List findByTagFQN(@Bind("tagFQN") String tagFQN); - - @SqlQuery("SELECT json FROM recognizer_feedback_entity WHERE status = :status") - List findByStatus(@Bind("status") String status); - - @SqlQuery("SELECT count(id) FROM recognizer_feedback_entity") - int count(); - - @SqlUpdate("DELETE FROM recognizer_feedback_entity WHERE id = :id") - void delete(@BindUUID("id") UUID id); - } - - class ExecutionTrendRow { - private String dateKey; - private String status; - private Integer count; - - public ExecutionTrendRow() {} - - public ExecutionTrendRow(String dateKey, String status, Integer count) { - this.dateKey = dateKey; - this.status = status; - this.count = count; - } - - public String getDateKey() { - return dateKey; - } - - public void setDateKey(String dateKey) { - this.dateKey = dateKey; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public Integer getCount() { - return count; - } - - public void setCount(Integer count) { - this.count = count; - } - } - - class ExecutionTrendRowMapper implements RowMapper { - @Override - public ExecutionTrendRow map(ResultSet rs, StatementContext ctx) throws SQLException { - ExecutionTrendRow row = new ExecutionTrendRow(); - row.setDateKey(rs.getString("date_key")); - row.setStatus(rs.getString("status")); - row.setCount(rs.getInt("count")); - return row; - } - } - - class RuntimeTrendRow { - private String dateKey; - private Long firstTimestamp; - private Double maxRuntime; - private Double minRuntime; - private Double avgRuntime; - private Integer totalPipelines; - - public RuntimeTrendRow() {} - - public RuntimeTrendRow( - String dateKey, - Long firstTimestamp, - Double maxRuntime, - Double minRuntime, - Double avgRuntime, - Integer totalPipelines) { - this.dateKey = dateKey; - this.firstTimestamp = firstTimestamp; - this.maxRuntime = maxRuntime; - this.minRuntime = minRuntime; - this.avgRuntime = avgRuntime; - this.totalPipelines = totalPipelines; - } - - public String getDateKey() { - return dateKey; - } - - public void setDateKey(String dateKey) { - this.dateKey = dateKey; - } - - public Long getFirstTimestamp() { - return firstTimestamp; - } - - public void setFirstTimestamp(Long firstTimestamp) { - this.firstTimestamp = firstTimestamp; - } - - public Double getMaxRuntime() { - return maxRuntime; - } - - public void setMaxRuntime(Double maxRuntime) { - this.maxRuntime = maxRuntime; - } - - public Double getMinRuntime() { - return minRuntime; - } - - public void setMinRuntime(Double minRuntime) { - this.minRuntime = minRuntime; - } - - public Double getAvgRuntime() { - return avgRuntime; - } - - public void setAvgRuntime(Double avgRuntime) { - this.avgRuntime = avgRuntime; - } - - public Integer getTotalPipelines() { - return totalPipelines; - } - - public void setTotalPipelines(Integer totalPipelines) { - this.totalPipelines = totalPipelines; - } - } - - class RuntimeTrendRowMapper implements RowMapper { - @Override - public RuntimeTrendRow map(ResultSet rs, StatementContext ctx) throws SQLException { - RuntimeTrendRow row = new RuntimeTrendRow(); - row.setDateKey(rs.getString("date_key")); - row.setFirstTimestamp(rs.getLong("first_timestamp")); - row.setMaxRuntime(rs.getDouble("max_runtime")); - row.setMinRuntime(rs.getDouble("min_runtime")); - row.setAvgRuntime(rs.getDouble("avg_runtime")); - row.setTotalPipelines(rs.getInt("total_pipelines")); - return row; - } - } - - class ServiceBreakdownRow { - private String serviceType; - private Integer pipelineCount; - - public ServiceBreakdownRow() {} - - public ServiceBreakdownRow(String serviceType, Integer pipelineCount) { - this.serviceType = serviceType; - this.pipelineCount = pipelineCount; - } - - public String getServiceType() { - return serviceType; - } - - public void setServiceType(String serviceType) { - this.serviceType = serviceType; - } - - public Integer getPipelineCount() { - return pipelineCount; - } - - public void setPipelineCount(Integer pipelineCount) { - this.pipelineCount = pipelineCount; - } - } - - class ServiceBreakdownRowMapper implements RowMapper { - @Override - public ServiceBreakdownRow map(ResultSet rs, StatementContext ctx) throws SQLException { - ServiceBreakdownRow row = new ServiceBreakdownRow(); - row.setServiceType(rs.getString("service_type")); - row.setPipelineCount(rs.getInt("pipeline_count")); - return row; - } - } - - class PipelineMetricsRow { - private Integer totalPipelines; - private Integer activePipelines; - private Integer successfulPipelines; - private Integer failedPipelines; - - public PipelineMetricsRow() {} - - public PipelineMetricsRow( - Integer totalPipelines, - Integer activePipelines, - Integer successfulPipelines, - Integer failedPipelines) { - this.totalPipelines = totalPipelines; - this.activePipelines = activePipelines; - this.successfulPipelines = successfulPipelines; - this.failedPipelines = failedPipelines; - } - - public Integer getTotalPipelines() { - return totalPipelines; - } - - public void setTotalPipelines(Integer totalPipelines) { - this.totalPipelines = totalPipelines; - } - - public Integer getActivePipelines() { - return activePipelines; - } - - public void setActivePipelines(Integer activePipelines) { - this.activePipelines = activePipelines; - } - - public Integer getSuccessfulPipelines() { - return successfulPipelines; - } - - public void setSuccessfulPipelines(Integer successfulPipelines) { - this.successfulPipelines = successfulPipelines; - } - - public Integer getFailedPipelines() { - return failedPipelines; - } - - public void setFailedPipelines(Integer failedPipelines) { - this.failedPipelines = failedPipelines; - } - } - - class PipelineMetricsRowMapper implements RowMapper { - @Override - public PipelineMetricsRow map(ResultSet rs, StatementContext ctx) throws SQLException { - PipelineMetricsRow row = new PipelineMetricsRow(); - row.setTotalPipelines(rs.getInt("total_pipelines")); - row.setActivePipelines(rs.getInt("active_pipelines")); - row.setSuccessfulPipelines(rs.getInt("successful_pipelines")); - row.setFailedPipelines(rs.getInt("failed_pipelines")); - return row; - } - } - - class PipelineSummaryRow { - private String id; - private String json; - private String latestStatus; - - public PipelineSummaryRow() {} - - public PipelineSummaryRow(String id, String json, String latestStatus) { - this.id = id; - this.json = json; - this.latestStatus = latestStatus; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getJson() { - return json; - } - - public void setJson(String json) { - this.json = json; - } - - public String getLatestStatus() { - return latestStatus; - } - - public void setLatestStatus(String latestStatus) { - this.latestStatus = latestStatus; - } - } - - class PipelineSummaryRowMapper implements RowMapper { - @Override - public PipelineSummaryRow map(ResultSet rs, StatementContext ctx) throws SQLException { - PipelineSummaryRow row = new PipelineSummaryRow(); - row.setId(rs.getString("id")); - row.setJson(rs.getString("json")); - row.setLatestStatus(rs.getString("latest_status")); - return row; - } - } - - interface AIApplicationDAO extends EntityDAO { - @Override - default String getTableName() { - return "ai_application_entity"; - } - - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.ai.AIApplication.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface LLMModelDAO extends EntityDAO { - @Override - default String getTableName() { - return "llm_model_entity"; - } - - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.ai.LLMModel.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface PromptTemplateDAO extends EntityDAO { - @Override - default String getTableName() { - return "prompt_template_entity"; - } - - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.ai.PromptTemplate.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface AgentExecutionDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "agent_execution_entity"; - } - - @Override - default String getPartitionFieldName() { - return "agentId"; - } - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO agent_execution_entity(json) VALUES (:json) AS new_data ON DUPLICATE KEY UPDATE json = new_data.json", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO agent_execution_entity(json) VALUES (:json::jsonb) ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json", - connectionType = POSTGRES) - void insertWithoutExtension( - @Define("table") String table, - @BindFQN("entityFQNHash") String entityFQNHash, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO agent_execution_entity(json) VALUES (:json) AS new_data ON DUPLICATE KEY UPDATE json = new_data.json", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO agent_execution_entity(json) VALUES (:json::jsonb) ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json", - connectionType = POSTGRES) - void insert( - @Define("table") String table, - @BindFQN("entityFQNHash") String entityFQNHash, - @Bind("extension") String extension, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM agent_execution_entity WHERE agentId = :agentId AND timestamp = :timestamp", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM agent_execution_entity WHERE agentId = :agentId AND timestamp = :timestamp", - connectionType = POSTGRES) - void deleteAtTimestamp( - @BindFQN("agentId") String agentId, - @Bind("extension") String extension, - @Bind("timestamp") Long timestamp); - - @SqlQuery("SELECT count(*) FROM agent_execution_entity ") - int listCount(@Define("cond") String condition); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM agent_execution_entity " - + "WHERE id IN (" - + " SELECT id FROM (" - + " SELECT ae.id FROM agent_execution_entity ae " - + " LEFT JOIN ai_application_entity ai ON ae.agentId = ai.id " - + " WHERE ai.id IS NULL " - + " LIMIT :limit" - + " ) sub" - + ")", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM agent_execution_entity " - + "WHERE id IN (" - + " SELECT ae.id FROM agent_execution_entity ae " - + " LEFT JOIN ai_application_entity ai ON ae.agentId = ai.id " - + " WHERE ai.id IS NULL " - + " LIMIT :limit" - + ")", - connectionType = POSTGRES) - int deleteOrphanedRecords(@Bind("limit") int limit); - } - - interface AIGovernancePolicyDAO - extends EntityDAO { - @Override - default String getTableName() { - return "ai_governance_policy_entity"; - } - - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.ai.AIGovernancePolicy.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface McpServerDAO extends EntityDAO { - @Override - default String getTableName() { - return "mcp_server_entity"; - } - - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.ai.McpServer.class; - } - - @Override - default String getNameHashColumn() { - return "fqnHash"; - } - } - - interface McpExecutionDAO extends EntityTimeSeriesDAO { - @Override - default String getTimeSeriesTableName() { - return "mcp_execution_entity"; - } - - @Override - default String getPartitionFieldName() { - return "serverId"; - } - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO
(json) VALUES (:json) AS new_data ON DUPLICATE KEY UPDATE json = new_data.json", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO
(json) VALUES (:json::jsonb) ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json", - connectionType = POSTGRES) - void insertWithoutExtension( - @Define("table") String table, - @BindFQN("entityFQNHash") String entityFQNHash, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO
(json) VALUES (:json) AS new_data ON DUPLICATE KEY UPDATE json = new_data.json", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO
(json) VALUES (:json::jsonb) ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json", - connectionType = POSTGRES) - void insert( - @Define("table") String table, - @BindFQN("entityFQNHash") String entityFQNHash, - @Bind("extension") String extension, - @Bind("jsonSchema") String jsonSchema, - @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = "DELETE FROM
WHERE serverId = :serverId AND timestamp = :timestamp", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "DELETE FROM
WHERE serverId = :serverId AND timestamp = :timestamp", - connectionType = POSTGRES) - void deleteAtTimestamp( - @Define("table") String table, - @Bind("serverId") String serverId, - @Bind("extension") String extension, - @Bind("timestamp") Long timestamp); - - @SqlQuery( - "SELECT json FROM
WHERE serverId = :serverId ORDER BY timestamp DESC LIMIT :limit") - List listByServerId( - @Define("table") String table, @Bind("serverId") String serverId, @Bind("limit") int limit); - - @SqlQuery("SELECT count(*) FROM
") - int listCount(@Define("table") String table, @Define("cond") String condition); - - @ConnectionAwareSqlUpdate( - value = "DELETE FROM
WHERE serverId = :serverId", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "DELETE FROM
WHERE serverId = :serverId", - connectionType = POSTGRES) - void deleteByServerId(@Define("table") String table, @Bind("serverId") String serverId); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM mcp_execution_entity " - + "WHERE id IN (" - + " SELECT id FROM (" - + " SELECT me.id FROM mcp_execution_entity me " - + " LEFT JOIN mcp_server_entity ms ON me.serverId = ms.id " - + " WHERE ms.id IS NULL " - + " LIMIT :limit" - + " ) sub" - + ")", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM mcp_execution_entity " - + "WHERE id IN (" - + " SELECT me.id FROM mcp_execution_entity me " - + " LEFT JOIN mcp_server_entity ms ON me.serverId = ms.id " - + " WHERE ms.id IS NULL " - + " LIMIT :limit" - + ")", - connectionType = POSTGRES) - int deleteOrphanedRecords(@Bind("limit") int limit); - } - - interface LLMServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "llm_service_entity"; - } - - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.services.LLMService.class; - } - } - - interface McpServiceDAO extends EntityDAO { - @Override - default String getTableName() { - return "mcp_service_entity"; - } - - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.services.McpService.class; - } - - @Override - default String getNameHashColumn() { - return "nameHash"; - } - } - - /** DAO for distributed search index jobs */ - interface SearchIndexJobDAO { - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_job (id, status, jobConfiguration, targetIndexPrefix, totalRecords, " - + "processedRecords, successRecords, failedRecords, stats, createdBy, createdAt, updatedAt, " - + "registrationDeadline) " - + "VALUES (:id, :status, :jobConfiguration, :targetIndexPrefix, :totalRecords, " - + ":processedRecords, :successRecords, :failedRecords, :stats, :createdBy, :createdAt, :updatedAt, " - + ":registrationDeadline)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_job (id, status, jobConfiguration, targetIndexPrefix, totalRecords, " - + "processedRecords, successRecords, failedRecords, stats, createdBy, createdAt, updatedAt, " - + "registrationDeadline) " - + "VALUES (:id, :status, :jobConfiguration::jsonb, :targetIndexPrefix, :totalRecords, " - + ":processedRecords, :successRecords, :failedRecords, :stats::jsonb, :createdBy, :createdAt, :updatedAt, " - + ":registrationDeadline)", - connectionType = POSTGRES) - void insert( - @Bind("id") String id, - @Bind("status") String status, - @Bind("jobConfiguration") String jobConfiguration, - @Bind("targetIndexPrefix") String targetIndexPrefix, - @Bind("totalRecords") long totalRecords, - @Bind("processedRecords") long processedRecords, - @Bind("successRecords") long successRecords, - @Bind("failedRecords") long failedRecords, - @Bind("stats") String stats, - @Bind("createdBy") String createdBy, - @Bind("createdAt") long createdAt, - @Bind("updatedAt") long updatedAt, - @Bind("registrationDeadline") Long registrationDeadline); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE search_index_job SET status = :status, processedRecords = :processedRecords, " - + "successRecords = :successRecords, failedRecords = :failedRecords, stats = :stats, " - + "startedAt = :startedAt, completedAt = :completedAt, updatedAt = :updatedAt, " - + "errorMessage = :errorMessage WHERE id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE search_index_job SET status = :status, processedRecords = :processedRecords, " - + "successRecords = :successRecords, failedRecords = :failedRecords, stats = :stats::jsonb, " - + "startedAt = :startedAt, completedAt = :completedAt, updatedAt = :updatedAt, " - + "errorMessage = :errorMessage WHERE id = :id", - connectionType = POSTGRES) - void update( - @Bind("id") String id, - @Bind("status") String status, - @Bind("processedRecords") long processedRecords, - @Bind("successRecords") long successRecords, - @Bind("failedRecords") long failedRecords, - @Bind("stats") String stats, - @Bind("startedAt") Long startedAt, - @Bind("completedAt") Long completedAt, - @Bind("updatedAt") long updatedAt, - @Bind("errorMessage") String errorMessage); - - @SqlQuery("SELECT * FROM search_index_job WHERE id = :id") - @RegisterRowMapper(SearchIndexJobMapper.class) - SearchIndexJobRecord findById(@Bind("id") String id); - - @SqlQuery("SELECT * FROM search_index_job WHERE status IN () ORDER BY createdAt DESC") - @RegisterRowMapper(SearchIndexJobMapper.class) - List findByStatuses(@BindList("statuses") List statuses); - - @SqlQuery( - "SELECT * FROM search_index_job WHERE status IN () ORDER BY createdAt DESC LIMIT :limit") - @RegisterRowMapper(SearchIndexJobMapper.class) - List findByStatusesWithLimit( - @BindList("statuses") List statuses, @Bind("limit") int limit); - - @SqlQuery("SELECT * FROM search_index_job ORDER BY createdAt DESC LIMIT :limit") - @RegisterRowMapper(SearchIndexJobMapper.class) - List listRecent(@Bind("limit") int limit); - - @SqlUpdate("DELETE FROM search_index_job WHERE id = :id") - void delete(@Bind("id") String id); - - @SqlUpdate( - "DELETE FROM search_index_job WHERE status IN ('COMPLETED', 'FAILED', 'STOPPED') AND completedAt < :before") - int deleteOldJobs(@Bind("before") long before); - - @SqlUpdate("DELETE FROM search_index_job") - void deleteAll(); - - @SqlUpdate( - "UPDATE search_index_job SET registeredServerCount = :serverCount, updatedAt = :updatedAt WHERE id = :id") - void updateRegisteredServerCount( - @Bind("id") String id, - @Bind("serverCount") int serverCount, - @Bind("updatedAt") long updatedAt); - - @SqlQuery("SELECT registrationDeadline FROM search_index_job WHERE id = :id") - Long getRegistrationDeadline(@Bind("id") String id); - - @SqlQuery("SELECT registeredServerCount FROM search_index_job WHERE id = :id") - Integer getRegisteredServerCount(@Bind("id") String id); - - /** Get IDs of currently running jobs - lightweight query for polling */ - @SqlQuery("SELECT id FROM search_index_job WHERE status = 'RUNNING'") - List getRunningJobIds(); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE search_index_job SET stagedIndexMapping = :stagedIndexMapping, updatedAt = :updatedAt WHERE id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE search_index_job SET stagedIndexMapping = :stagedIndexMapping::jsonb, updatedAt = :updatedAt WHERE id = :id", - connectionType = POSTGRES) - void updateStagedIndexMapping( - @Bind("id") String id, - @Bind("stagedIndexMapping") String stagedIndexMapping, - @Bind("updatedAt") long updatedAt); - - @SqlUpdate("UPDATE search_index_job SET updatedAt = :updatedAt WHERE id = :id") - void touchJob(@Bind("id") String id, @Bind("updatedAt") long updatedAt); - - /** Row mapper for SearchIndexJobRecord */ - class SearchIndexJobMapper implements RowMapper { - @Override - public SearchIndexJobRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new SearchIndexJobRecord( - rs.getString("id"), - rs.getString("status"), - rs.getString("jobConfiguration"), - rs.getString("targetIndexPrefix"), - rs.getString("stagedIndexMapping"), - rs.getLong("totalRecords"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getString("stats"), - rs.getString("createdBy"), - rs.getLong("createdAt"), - (Long) rs.getObject("startedAt"), - (Long) rs.getObject("completedAt"), - rs.getLong("updatedAt"), - rs.getString("errorMessage"), - (Long) rs.getObject("registrationDeadline"), - (Integer) rs.getObject("registeredServerCount")); - } - } - - /** Record for job data from DB */ - record SearchIndexJobRecord( - String id, - String status, - String jobConfiguration, - String targetIndexPrefix, - String stagedIndexMapping, - long totalRecords, - long processedRecords, - long successRecords, - long failedRecords, - String stats, - String createdBy, - long createdAt, - Long startedAt, - Long completedAt, - long updatedAt, - String errorMessage, - Long registrationDeadline, - Integer registeredServerCount) {} - } - - /** DAO for distributed search index partitions */ - interface SearchIndexPartitionDAO { - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_partition (id, jobId, entityType, partitionIndex, rangeStart, rangeEnd, " - + "estimatedCount, workUnits, priority, status, processingCursor, claimableAt) " - + "VALUES (:id, :jobId, :entityType, :partitionIndex, :rangeStart, :rangeEnd, " - + ":estimatedCount, :workUnits, :priority, :status, :cursor, :claimableAt)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_partition (id, jobId, entityType, partitionIndex, rangeStart, rangeEnd, " - + "estimatedCount, workUnits, priority, status, processingCursor, claimableAt) " - + "VALUES (:id, :jobId, :entityType, :partitionIndex, :rangeStart, :rangeEnd, " - + ":estimatedCount, :workUnits, :priority, :status, :cursor, :claimableAt)", - connectionType = POSTGRES) - void insert( - @Bind("id") String id, - @Bind("jobId") String jobId, - @Bind("entityType") String entityType, - @Bind("partitionIndex") int partitionIndex, - @Bind("rangeStart") long rangeStart, - @Bind("rangeEnd") long rangeEnd, - @Bind("estimatedCount") long estimatedCount, - @Bind("workUnits") long workUnits, - @Bind("priority") int priority, - @Bind("status") String status, - @Bind("cursor") long cursor, - @Bind("claimableAt") long claimableAt); - - @SqlUpdate( - "UPDATE search_index_partition SET status = :status, processingCursor = :cursor, " - + "processedCount = :processedCount, successCount = :successCount, failedCount = :failedCount, " - + "assignedServer = :assignedServer, claimedAt = :claimedAt, startedAt = :startedAt, " - + "completedAt = :completedAt, lastUpdateAt = :lastUpdateAt, lastError = :lastError, " - + "retryCount = :retryCount WHERE id = :id") - void update( - @Bind("id") String id, - @Bind("status") String status, - @Bind("cursor") long cursor, - @Bind("processedCount") long processedCount, - @Bind("successCount") long successCount, - @Bind("failedCount") long failedCount, - @Bind("assignedServer") String assignedServer, - @Bind("claimedAt") Long claimedAt, - @Bind("startedAt") Long startedAt, - @Bind("completedAt") Long completedAt, - @Bind("lastUpdateAt") Long lastUpdateAt, - @Bind("lastError") String lastError, - @Bind("retryCount") int retryCount); - - @SqlUpdate( - "UPDATE search_index_partition SET processingCursor = :cursor, processedCount = :processedCount, " - + "successCount = :successCount, failedCount = :failedCount, lastUpdateAt = :lastUpdateAt " - + "WHERE id = :id") - void updateProgress( - @Bind("id") String id, - @Bind("cursor") long cursor, - @Bind("processedCount") long processedCount, - @Bind("successCount") long successCount, - @Bind("failedCount") long failedCount, - @Bind("lastUpdateAt") long lastUpdateAt); - - @SqlUpdate("UPDATE search_index_partition SET lastUpdateAt = :lastUpdateAt WHERE id = :id") - void updateHeartbeat(@Bind("id") String id, @Bind("lastUpdateAt") long lastUpdateAt); - - @SqlQuery("SELECT * FROM search_index_partition WHERE id = :id") - @RegisterRowMapper(SearchIndexPartitionMapper.class) - SearchIndexPartitionRecord findById(@Bind("id") String id); - - @SqlQuery( - "SELECT * FROM search_index_partition WHERE jobId = :jobId ORDER BY priority DESC, entityType, partitionIndex") - @RegisterRowMapper(SearchIndexPartitionMapper.class) - List findByJobId(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT * FROM search_index_partition WHERE jobId = :jobId AND status = 'PENDING' " - + "AND claimableAt <= :now " - + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED") - @RegisterRowMapper(SearchIndexPartitionMapper.class) - SearchIndexPartitionRecord findNextPendingPartitionForUpdate( - @Bind("jobId") String jobId, @Bind("now") long now); - - @SqlUpdate( - "UPDATE search_index_partition SET status = 'PROCESSING', " - + "assignedServer = :serverId, claimedAt = :now, startedAt = :now, lastUpdateAt = :now " - + "WHERE id = :partitionId AND status = 'PENDING'") - int claimPartitionById( - @Bind("partitionId") String partitionId, - @Bind("serverId") String serverId, - @Bind("now") long now); - - /** - * Atomically claim the next available partition using UPDATE with subquery. - * MySQL requires a JOIN-based approach since it doesn't allow subquery referencing same table. - * PostgreSQL can use direct subquery approach. - * Only claims partitions where claimableAt <= now (for staggered release). - */ - @ConnectionAwareSqlUpdate( - value = - "UPDATE search_index_partition p " - + "JOIN (SELECT id FROM search_index_partition WHERE jobId = :jobId AND status = 'PENDING' " - + "AND claimableAt <= :now " - + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED) t ON p.id = t.id " - + "SET p.status = 'PROCESSING', p.assignedServer = :serverId, p.claimedAt = :now, " - + "p.startedAt = :now, p.lastUpdateAt = :now", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE search_index_partition SET status = 'PROCESSING', " - + "assignedServer = :serverId, claimedAt = :now, startedAt = :now, lastUpdateAt = :now " - + "WHERE id = (SELECT id FROM search_index_partition WHERE jobId = :jobId AND status = 'PENDING' " - + "AND claimableAt <= :now " - + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED)", - connectionType = POSTGRES) - int claimNextPartitionAtomic( - @Bind("jobId") String jobId, @Bind("serverId") String serverId, @Bind("now") long now); - - @SqlQuery( - "SELECT * FROM search_index_partition WHERE jobId = :jobId AND status = 'PROCESSING' " - + "AND assignedServer = :serverId AND claimedAt = :claimedAt " - + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1") - @RegisterRowMapper(SearchIndexPartitionMapper.class) - SearchIndexPartitionRecord findLatestClaimedPartition( - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("claimedAt") long claimedAt); - - @SqlQuery( - "SELECT * FROM search_index_partition WHERE jobId = :jobId AND status = :status " - + "ORDER BY priority DESC, entityType, partitionIndex") - @RegisterRowMapper(SearchIndexPartitionMapper.class) - List findByJobIdAndStatus( - @Bind("jobId") String jobId, @Bind("status") String status); - - /** Count how many partitions a server currently has in PROCESSING status for a job */ - @SqlQuery( - "SELECT COUNT(*) FROM search_index_partition " - + "WHERE jobId = :jobId AND status = 'PROCESSING' AND assignedServer = :serverId") - int countInFlightPartitions(@Bind("jobId") String jobId, @Bind("serverId") String serverId); - - /** Count total PENDING partitions for a job */ - @SqlQuery( - "SELECT COUNT(*) FROM search_index_partition WHERE jobId = :jobId AND status = 'PENDING'") - int countPendingPartitions(@Bind("jobId") String jobId); - - /** Count total partitions for a job */ - @SqlQuery("SELECT COUNT(*) FROM search_index_partition WHERE jobId = :jobId") - int countTotalPartitions(@Bind("jobId") String jobId); - - /** Count partitions claimed by a specific server (PROCESSING or COMPLETED) */ - @SqlQuery( - "SELECT COUNT(*) FROM search_index_partition " - + "WHERE jobId = :jobId AND assignedServer = :serverId") - int countPartitionsClaimedByServer( - @Bind("jobId") String jobId, @Bind("serverId") String serverId); - - /** Count distinct servers that have claimed partitions for a job */ - @SqlQuery( - "SELECT COUNT(DISTINCT assignedServer) FROM search_index_partition " - + "WHERE jobId = :jobId AND assignedServer IS NOT NULL") - int countParticipatingServers(@Bind("jobId") String jobId); - - /** - * Reclaim stale partitions that can still be retried (under max retry limit). - * Returns the count of partitions reset to PENDING. - */ - @SqlUpdate( - "UPDATE search_index_partition SET status = 'PENDING', assignedServer = NULL, claimedAt = NULL, " - + "retryCount = retryCount + 1, lastError = 'Reclaimed due to stale heartbeat' " - + "WHERE jobId = :jobId AND status = 'PROCESSING' AND lastUpdateAt < :staleThreshold " - + "AND retryCount < :maxRetries") - int reclaimStalePartitionsForRetry( - @Bind("jobId") String jobId, - @Bind("staleThreshold") long staleThreshold, - @Bind("maxRetries") int maxRetries); - - /** - * Mark stale partitions that have exceeded retry limit as FAILED. - * Returns the count of partitions marked as failed. - */ - @SqlUpdate( - "UPDATE search_index_partition SET status = 'FAILED', " - + "lastError = 'Exceeded max retries after stale heartbeat', completedAt = :now " - + "WHERE jobId = :jobId AND status = 'PROCESSING' AND lastUpdateAt < :staleThreshold " - + "AND retryCount >= :maxRetries") - int failStalePartitionsExceedingRetries( - @Bind("jobId") String jobId, - @Bind("staleThreshold") long staleThreshold, - @Bind("maxRetries") int maxRetries, - @Bind("now") long now); - - @SqlUpdate( - "UPDATE search_index_partition SET status = 'CANCELLED' WHERE jobId = :jobId AND status = 'PENDING'") - int cancelPendingPartitions(@Bind("jobId") String jobId); - - @SqlUpdate( - "UPDATE search_index_partition SET status = 'CANCELLED', " - + "lastError = 'Stopped by user', completedAt = :now, lastUpdateAt = :now " - + "WHERE jobId = :jobId AND status IN ('PENDING','PROCESSING')") - int cancelInFlightPartitions(@Bind("jobId") String jobId, @Bind("now") long now); - - /** - * Status-guarded update: only mutates the row when it is still PROCESSING. Used by - * completion / failure paths so a late-arriving worker write cannot revert a CANCELLED - * row (set by requestStop) back to COMPLETED/FAILED. Returns the number of rows - * updated — 0 means another writer (typically requestStop) already moved the row to - * a terminal state and the caller should treat its update as a no-op. - */ - @SqlUpdate( - "UPDATE search_index_partition SET status = :status, processingCursor = :cursor, " - + "processedCount = :processedCount, successCount = :successCount, failedCount = :failedCount, " - + "assignedServer = :assignedServer, claimedAt = :claimedAt, startedAt = :startedAt, " - + "completedAt = :completedAt, lastUpdateAt = :lastUpdateAt, lastError = :lastError, " - + "retryCount = :retryCount WHERE id = :id AND status = 'PROCESSING'") - int updateIfProcessing( - @Bind("id") String id, - @Bind("status") String status, - @Bind("cursor") long cursor, - @Bind("processedCount") long processedCount, - @Bind("successCount") long successCount, - @Bind("failedCount") long failedCount, - @Bind("assignedServer") String assignedServer, - @Bind("claimedAt") Long claimedAt, - @Bind("startedAt") Long startedAt, - @Bind("completedAt") Long completedAt, - @Bind("lastUpdateAt") Long lastUpdateAt, - @Bind("lastError") String lastError, - @Bind("retryCount") int retryCount); - - @SqlQuery( - "SELECT * FROM search_index_partition WHERE jobId = :jobId AND status = 'PROCESSING' " - + "AND lastUpdateAt < :staleThreshold") - @RegisterRowMapper(SearchIndexPartitionMapper.class) - List findStalePartitions( - @Bind("jobId") String jobId, @Bind("staleThreshold") long staleThreshold); - - @SqlQuery( - "SELECT entityType, " - + "SUM(estimatedCount) as totalRecords, " - + "SUM(processedCount) as processedRecords, " - + "SUM(successCount) as successRecords, " - + "SUM(failedCount) as failedRecords, " - + "COUNT(*) as totalPartitions, " - + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " - + "SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) as failedPartitions " - + "FROM search_index_partition WHERE jobId = :jobId GROUP BY entityType") - @RegisterRowMapper(EntityStatsMapper.class) - List getEntityStats(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT " - + "SUM(estimatedCount) as totalRecords, " - + "SUM(processedCount) as processedRecords, " - + "SUM(successCount) as successRecords, " - + "SUM(failedCount) as failedRecords, " - + "COUNT(*) as totalPartitions, " - + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " - + "SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) as failedPartitions, " - + "SUM(CASE WHEN status = 'PENDING' THEN 1 ELSE 0 END) as pendingPartitions, " - + "SUM(CASE WHEN status = 'PROCESSING' THEN 1 ELSE 0 END) as processingPartitions " - + "FROM search_index_partition WHERE jobId = :jobId") - @RegisterRowMapper(AggregatedStatsMapper.class) - AggregatedStatsRecord getAggregatedStats(@Bind("jobId") String jobId); - - @SqlUpdate("DELETE FROM search_index_partition WHERE jobId = :jobId") - void deleteByJobId(@Bind("jobId") String jobId); - - @SqlUpdate("DELETE FROM search_index_partition") - void deleteAll(); - - @SqlQuery( - "SELECT assignedServer, " - + "SUM(processedCount) as processedRecords, " - + "SUM(successCount) as successRecords, " - + "SUM(failedCount) as failedRecords, " - + "COUNT(*) as totalPartitions, " - + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " - + "SUM(CASE WHEN status = 'PROCESSING' THEN 1 ELSE 0 END) as processingPartitions " - + "FROM search_index_partition WHERE jobId = :jobId AND assignedServer IS NOT NULL " - + "GROUP BY assignedServer") - @RegisterRowMapper(ServerStatsMapper.class) - List getServerStats(@Bind("jobId") String jobId); - - /** Row mapper for partition records */ - class SearchIndexPartitionMapper implements RowMapper { - @Override - public SearchIndexPartitionRecord map(ResultSet rs, StatementContext ctx) - throws SQLException { - return new SearchIndexPartitionRecord( - rs.getString("id"), - rs.getString("jobId"), - rs.getString("entityType"), - rs.getInt("partitionIndex"), - rs.getLong("rangeStart"), - rs.getLong("rangeEnd"), - rs.getLong("estimatedCount"), - rs.getLong("workUnits"), - rs.getInt("priority"), - rs.getString("status"), - rs.getLong("processingCursor"), - rs.getLong("processedCount"), - rs.getLong("successCount"), - rs.getLong("failedCount"), - rs.getString("assignedServer"), - (Long) rs.getObject("claimedAt"), - (Long) rs.getObject("startedAt"), - (Long) rs.getObject("completedAt"), - (Long) rs.getObject("lastUpdateAt"), - rs.getString("lastError"), - rs.getInt("retryCount"), - rs.getLong("claimableAt")); - } - } - - /** Row mapper for entity stats */ - class EntityStatsMapper implements RowMapper { - @Override - public EntityStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new EntityStatsRecord( - rs.getString("entityType"), - rs.getLong("totalRecords"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getInt("totalPartitions"), - rs.getInt("completedPartitions"), - rs.getInt("failedPartitions")); - } - } - - /** Row mapper for aggregated stats */ - class AggregatedStatsMapper implements RowMapper { - @Override - public AggregatedStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new AggregatedStatsRecord( - rs.getLong("totalRecords"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getInt("totalPartitions"), - rs.getInt("completedPartitions"), - rs.getInt("failedPartitions"), - rs.getInt("pendingPartitions"), - rs.getInt("processingPartitions")); - } - } - - /** Record for partition data from DB */ - record SearchIndexPartitionRecord( - String id, - String jobId, - String entityType, - int partitionIndex, - long rangeStart, - long rangeEnd, - long estimatedCount, - long workUnits, - int priority, - String status, - long cursor, - long processedCount, - long successCount, - long failedCount, - String assignedServer, - Long claimedAt, - Long startedAt, - Long completedAt, - Long lastUpdateAt, - String lastError, - int retryCount, - long claimableAt) {} - - /** Record for entity stats aggregation */ - record EntityStatsRecord( - String entityType, - long totalRecords, - long processedRecords, - long successRecords, - long failedRecords, - int totalPartitions, - int completedPartitions, - int failedPartitions) {} - - /** Record for overall job stats aggregation */ - record AggregatedStatsRecord( - long totalRecords, - long processedRecords, - long successRecords, - long failedRecords, - int totalPartitions, - int completedPartitions, - int failedPartitions, - int pendingPartitions, - int processingPartitions) {} - - /** Record for per-server stats aggregation */ - record ServerStatsRecord( - String serverId, - long processedRecords, - long successRecords, - long failedRecords, - int totalPartitions, - int completedPartitions, - int processingPartitions) {} - - /** Row mapper for server stats */ - class ServerStatsMapper implements RowMapper { - @Override - public ServerStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new ServerStatsRecord( - rs.getString("assignedServer"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getInt("totalPartitions"), - rs.getInt("completedPartitions"), - rs.getInt("processingPartitions")); - } - } - - /** - * Record for partition quota statistics used in fair distribution. - * Includes both partition-count and work-based metrics for fair load balancing. - * - *

Work-based distribution ensures servers with high-record partitions don't - * monopolize the workload, even if partition counts appear balanced. - */ - record PartitionQuotaStats( - int inFlightCount, - int totalPartitions, - int claimedByServer, - int participatingServers, - int pendingPartitions, - long totalWorkUnits, - long workClaimedByServer, - long pendingWorkUnits) {} - - /** Row mapper for partition quota stats */ - class PartitionQuotaStatsMapper implements RowMapper { - @Override - public PartitionQuotaStats map(ResultSet rs, StatementContext ctx) throws SQLException { - return new PartitionQuotaStats( - rs.getInt("inFlightCount"), - rs.getInt("totalPartitions"), - rs.getInt("claimedByServer"), - rs.getInt("participatingServers"), - rs.getInt("pendingPartitions"), - rs.getLong("totalWorkUnits"), - rs.getLong("workClaimedByServer"), - rs.getLong("pendingWorkUnits")); - } - } - - /** - * Get all quota-related statistics in a single query for fair partition distribution. - * Includes both partition-count and work-based metrics. - */ - @SqlQuery( - "SELECT " - + "SUM(CASE WHEN status = 'PROCESSING' AND assignedServer = :serverId THEN 1 ELSE 0 END) as inFlightCount, " - + "COUNT(*) as totalPartitions, " - + "SUM(CASE WHEN assignedServer = :serverId THEN 1 ELSE 0 END) as claimedByServer, " - + "COUNT(DISTINCT CASE WHEN assignedServer IS NOT NULL THEN assignedServer END) as participatingServers, " - + "SUM(CASE WHEN status = 'PENDING' THEN 1 ELSE 0 END) as pendingPartitions, " - + "COALESCE(SUM(workUnits), 0) as totalWorkUnits, " - + "COALESCE(SUM(CASE WHEN assignedServer = :serverId THEN workUnits ELSE 0 END), 0) as workClaimedByServer, " - + "COALESCE(SUM(CASE WHEN status = 'PENDING' THEN workUnits ELSE 0 END), 0) as pendingWorkUnits " - + "FROM search_index_partition WHERE jobId = :jobId") - @RegisterRowMapper(PartitionQuotaStatsMapper.class) - PartitionQuotaStats getQuotaStats( - @Bind("jobId") String jobId, @Bind("serverId") String serverId); - - /** Get distinct servers that have claimed partitions for a job */ - @SqlQuery( - "SELECT DISTINCT assignedServer FROM search_index_partition " - + "WHERE jobId = :jobId AND assignedServer IS NOT NULL") - List getAssignedServers(@Bind("jobId") String jobId); - } - - /** DAO for distributed reindex lock */ - interface SearchReindexLockDAO { - - @SqlUpdate( - "INSERT INTO search_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " - + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt)") - void insert( - @Bind("lockKey") String lockKey, - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("acquiredAt") long acquiredAt, - @Bind("lastHeartbeat") long lastHeartbeat, - @Bind("expiresAt") long expiresAt); - - @ConnectionAwareSqlUpdate( - value = - "INSERT IGNORE INTO search_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " - + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " - + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt) " - + "ON CONFLICT (lockKey) DO NOTHING", - connectionType = POSTGRES) - int insertIfNotExists( - @Bind("lockKey") String lockKey, - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("acquiredAt") long acquiredAt, - @Bind("lastHeartbeat") long lastHeartbeat, - @Bind("expiresAt") long expiresAt); - - @SqlUpdate( - "UPDATE search_reindex_lock SET lastHeartbeat = :lastHeartbeat, expiresAt = :expiresAt " - + "WHERE lockKey = :lockKey AND jobId = :jobId") - int updateHeartbeat( - @Bind("lockKey") String lockKey, - @Bind("jobId") String jobId, - @Bind("lastHeartbeat") long lastHeartbeat, - @Bind("expiresAt") long expiresAt); - - @SqlQuery("SELECT * FROM search_reindex_lock WHERE lockKey = :lockKey") - @RegisterRowMapper(SearchReindexLockMapper.class) - SearchReindexLockRecord findByKey(@Bind("lockKey") String lockKey); - - @SqlUpdate("DELETE FROM search_reindex_lock WHERE lockKey = :lockKey") - void delete(@Bind("lockKey") String lockKey); - - @SqlUpdate("DELETE FROM search_reindex_lock WHERE lockKey = :lockKey AND jobId = :jobId") - int deleteByKeyAndJob(@Bind("lockKey") String lockKey, @Bind("jobId") String jobId); - - @SqlUpdate("DELETE FROM search_reindex_lock WHERE expiresAt < :now") - int deleteExpiredLocks(@Bind("now") long now); - - /** - * Try to acquire a lock using atomic INSERT with conflict handling. Returns true if lock was - * acquired. - * - *

Uses database-level atomicity to prevent race conditions: - *

    - *
  • PostgreSQL: INSERT ... ON CONFLICT DO NOTHING - *
  • MySQL: INSERT IGNORE - *
- * - *

If the insert fails due to a conflict, we check if the existing lock is expired and retry - * once after cleaning it up. - */ - default boolean tryAcquireLock( - String lockKey, String jobId, String serverId, long acquiredAt, long expiresAt) { - // First delete any expired locks - deleteExpiredLocks(System.currentTimeMillis()); - - // Atomically try to insert the lock - returns 1 if inserted, 0 if conflict - int inserted = insertIfNotExists(lockKey, jobId, serverId, acquiredAt, acquiredAt, expiresAt); - if (inserted > 0) { - return true; // Lock acquired successfully - } - - // Insert failed due to conflict - check if existing lock is expired - SearchReindexLockRecord existing = findByKey(lockKey); - if (existing != null && existing.isExpired()) { - // Lock is expired, delete it and retry once - delete(lockKey); - inserted = insertIfNotExists(lockKey, jobId, serverId, acquiredAt, acquiredAt, expiresAt); - return inserted > 0; - } - - // Lock is held by another active job - return false; - } - - /** Release a lock for a specific job */ - default void releaseLock(String lockKey, String jobId) { - deleteByKeyAndJob(lockKey, jobId); - } - - @SqlUpdate( - "UPDATE search_reindex_lock SET jobId = :toJobId, serverId = :serverId, " - + "lastHeartbeat = :heartbeat, expiresAt = :expiresAt " - + "WHERE lockKey = :lockKey AND jobId = :fromJobId") - int updateLockOwner( - @Bind("lockKey") String lockKey, - @Bind("fromJobId") String fromJobId, - @Bind("toJobId") String toJobId, - @Bind("serverId") String serverId, - @Bind("heartbeat") long heartbeat, - @Bind("expiresAt") long expiresAt); - - /** Atomically transfer a lock from one job to another */ - default boolean transferLock( - String lockKey, - String fromJobId, - String toJobId, - String serverId, - long heartbeat, - long expiresAt) { - int updated = updateLockOwner(lockKey, fromJobId, toJobId, serverId, heartbeat, expiresAt); - return updated > 0; - } - - /** Refresh a lock's heartbeat and expiration */ - default boolean refreshLock( - String lockKey, String jobId, String serverId, long heartbeat, long expiresAt) { - int updated = updateHeartbeat(lockKey, jobId, heartbeat, expiresAt); - return updated > 0; - } - - /** Clean up expired locks */ - default int cleanupExpiredLocks(long expirationThreshold) { - return deleteExpiredLocks(expirationThreshold); - } - - /** Get lock info for a specific lock key */ - default LockInfo getLockInfo(String lockKey) { - SearchReindexLockRecord record = findByKey(lockKey); - if (record == null) { - return null; - } - return new LockInfo( - record.lockKey(), - record.jobId(), - record.serverId(), - record.acquiredAt(), - record.lastHeartbeat(), - record.expiresAt()); - } - - /** Simple record for lock information */ - record LockInfo( - String lockKey, - String jobId, - String serverId, - long acquiredAt, - long lastHeartbeat, - long expiresAt) { - - public boolean isExpired() { - return System.currentTimeMillis() > expiresAt; - } - } - - /** Row mapper for lock records */ - class SearchReindexLockMapper implements RowMapper { - @Override - public SearchReindexLockRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new SearchReindexLockRecord( - rs.getString("lockKey"), - rs.getString("jobId"), - rs.getString("serverId"), - rs.getLong("acquiredAt"), - rs.getLong("lastHeartbeat"), - rs.getLong("expiresAt")); - } - } - - /** Record for lock data from DB */ - record SearchReindexLockRecord( - String lockKey, - String jobId, - String serverId, - long acquiredAt, - long lastHeartbeat, - long expiresAt) { - - public boolean isExpired() { - return System.currentTimeMillis() > expiresAt; - } - } - } - - /** DAO for search index failure records */ - interface SearchIndexFailureDAO { - - /** Bean class for @BindBean compatibility (records use id() not getId()) */ - @lombok.Getter - @lombok.AllArgsConstructor - class SearchIndexFailureRecord { - private final String id; - private final String jobId; - private final String serverId; - private final String entityType; - private final String entityId; - private final String entityFqn; - private final String failureStage; - private final String errorMessage; - private final String stackTrace; - private final long timestamp; - } - - @SqlUpdate( - "INSERT INTO search_index_failures (id, jobId, serverId, entityType, entityId, entityFqn, " - + "failureStage, errorMessage, stackTrace, timestamp) " - + "VALUES (:id, :jobId, :serverId, :entityType, :entityId, :entityFqn, " - + ":failureStage, :errorMessage, :stackTrace, :timestamp)") - void insert( - @Bind("id") String id, - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("entityType") String entityType, - @Bind("entityId") String entityId, - @Bind("entityFqn") String entityFqn, - @Bind("failureStage") String failureStage, - @Bind("errorMessage") String errorMessage, - @Bind("stackTrace") String stackTrace, - @Bind("timestamp") long timestamp); - - @SqlBatch( - "INSERT INTO search_index_failures (id, jobId, serverId, entityType, entityId, entityFqn, " - + "failureStage, errorMessage, stackTrace, timestamp) " - + "VALUES (:id, :jobId, :serverId, :entityType, :entityId, :entityFqn, " - + ":failureStage, :errorMessage, :stackTrace, :timestamp)") - void insertBatch(@BindBean List failures); - - @SqlQuery( - "SELECT * FROM search_index_failures WHERE serverId = :serverId " - + "ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") - @RegisterRowMapper(SearchIndexFailureMapper.class) - List findByServerId( - @Bind("serverId") String serverId, @Bind("limit") int limit, @Bind("offset") int offset); - - @SqlQuery("SELECT COUNT(*) FROM search_index_failures WHERE serverId = :serverId") - int countByServerId(@Bind("serverId") String serverId); - - @SqlQuery( - "SELECT * FROM search_index_failures WHERE jobId = :jobId " - + "ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") - @RegisterRowMapper(SearchIndexFailureMapper.class) - List findByJobId( - @Bind("jobId") String jobId, @Bind("limit") int limit, @Bind("offset") int offset); - - @SqlQuery("SELECT COUNT(*) FROM search_index_failures WHERE jobId = :jobId") - int countByJobId(@Bind("jobId") String jobId); - - /** - * Count only real failures for a job, excluding {@code READER_RELATIONSHIP_WARNING} rows — - * stale-relationship warnings are recorded for visibility but are not failures. - */ - @SqlQuery( - "SELECT COUNT(*) FROM search_index_failures WHERE jobId = :jobId " - + "AND failureStage <> 'READER_RELATIONSHIP_WARNING'") - int countFailuresByJobId(@Bind("jobId") String jobId); - - @SqlUpdate("DELETE FROM search_index_failures WHERE timestamp < :cutoffTime") - int deleteOlderThan(@Bind("cutoffTime") long cutoffTime); - - @SqlUpdate("DELETE FROM search_index_failures WHERE jobId = :jobId") - int deleteByJobId(@Bind("jobId") String jobId); - - @SqlUpdate("DELETE FROM search_index_failures") - int deleteAll(); - - @SqlQuery("SELECT COUNT(*) FROM search_index_failures") - int countAll(); - - @SqlQuery( - "SELECT * FROM search_index_failures ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") - @RegisterRowMapper(SearchIndexFailureMapper.class) - List findAll(@Bind("limit") int limit, @Bind("offset") int offset); - - @SqlQuery("SELECT COUNT(*) FROM search_index_failures WHERE entityType = :entityType") - int countByEntityType(@Bind("entityType") String entityType); - - @SqlQuery( - "SELECT * FROM search_index_failures WHERE entityType = :entityType " - + "ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") - @RegisterRowMapper(SearchIndexFailureMapper.class) - List findByEntityType( - @Bind("entityType") String entityType, - @Bind("limit") int limit, - @Bind("offset") int offset); - - class SearchIndexFailureMapper implements RowMapper { - @Override - public SearchIndexFailureRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new SearchIndexFailureRecord( - rs.getString("id"), - rs.getString("jobId"), - rs.getString("serverId"), - rs.getString("entityType"), - rs.getString("entityId"), - rs.getString("entityFqn"), - rs.getString("failureStage"), - rs.getString("errorMessage"), - rs.getString("stackTrace"), - rs.getLong("timestamp")); - } - } - } - - /** DAO for incremental search retry queue records. */ - interface SearchIndexRetryQueueDAO { - - @lombok.Getter - @lombok.AllArgsConstructor - class SearchIndexRetryRecord { - private final String entityId; - private final String entityFqn; - private final String failureReason; - private final String status; - private final String entityType; - private final int retryCount; - private final java.sql.Timestamp claimedAt; - } - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_retry_queue (entityId, entityFqn, failureReason, status, entityType) " - + "VALUES (:entityId, :entityFqn, :failureReason, :status, :entityType) " - + "ON DUPLICATE KEY UPDATE failureReason = VALUES(failureReason), status = VALUES(status), entityType = VALUES(entityType)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_retry_queue (entityId, entityFqn, failureReason, status, entityType) " - + "VALUES (:entityId, :entityFqn, :failureReason, :status, :entityType) " - + "ON CONFLICT (entityId, entityFqn) DO UPDATE SET " - + "failureReason = EXCLUDED.failureReason, status = EXCLUDED.status, entityType = EXCLUDED.entityType", - connectionType = POSTGRES) - void upsert( - @Bind("entityId") String entityId, - @Bind("entityFqn") String entityFqn, - @Bind("failureReason") String failureReason, - @Bind("status") String status, - @Bind("entityType") String entityType); - - @SqlQuery( - "SELECT entityId, entityFqn, failureReason, status, entityType, retryCount, claimedAt " - + "FROM search_index_retry_queue WHERE status = :status LIMIT :limit") - @RegisterRowMapper(SearchIndexRetryRecordMapper.class) - List findByStatus( - @Bind("status") String status, @Bind("limit") int limit); - - @SqlQuery( - "SELECT entityId, entityFqn, failureReason, status, entityType, retryCount, claimedAt " - + "FROM search_index_retry_queue WHERE status IN () LIMIT :limit") - @RegisterRowMapper(SearchIndexRetryRecordMapper.class) - List findByStatuses( - @BindList("statuses") List statuses, @Bind("limit") int limit); - - @SqlUpdate( - "UPDATE search_index_retry_queue SET status = :newStatus " - + "WHERE entityId = :entityId AND entityFqn = :entityFqn AND status = :currentStatus") - int updateStatusIfCurrent( - @Bind("entityId") String entityId, - @Bind("entityFqn") String entityFqn, - @Bind("currentStatus") String currentStatus, - @Bind("newStatus") String newStatus); - - @SqlUpdate( - "UPDATE search_index_retry_queue SET status = :status, failureReason = :failureReason " - + "WHERE entityId = :entityId AND entityFqn = :entityFqn") - int updateFailureAndStatus( - @Bind("entityId") String entityId, - @Bind("entityFqn") String entityFqn, - @Bind("failureReason") String failureReason, - @Bind("status") String status); - - @SqlUpdate( - "UPDATE search_index_retry_queue SET status = :status " - + "WHERE entityId = :entityId AND entityFqn = :entityFqn") - int updateStatus( - @Bind("entityId") String entityId, - @Bind("entityFqn") String entityFqn, - @Bind("status") String status); - - @SqlUpdate( - "DELETE FROM search_index_retry_queue WHERE entityId = :entityId AND entityFqn = :entityFqn") - int deleteByEntity(@Bind("entityId") String entityId, @Bind("entityFqn") String entityFqn); - - @SqlUpdate("DELETE FROM search_index_retry_queue WHERE status IN ()") - int deleteByStatuses(@BindList("statuses") List statuses); - - @SqlQuery("SELECT COUNT(*) FROM search_index_retry_queue WHERE status = :status") - int countByStatus(@Bind("status") String status); - - @SqlUpdate( - "UPDATE search_index_retry_queue SET status = 'IN_PROGRESS', claimedAt = NOW() " - + "WHERE entityId = :entityId AND entityFqn = :entityFqn AND status = :currentStatus") - int claimRecord( - @Bind("entityId") String entityId, - @Bind("entityFqn") String entityFqn, - @Bind("currentStatus") String currentStatus); - - @SqlUpdate( - "UPDATE search_index_retry_queue SET status = 'PENDING', claimedAt = NULL " - + "WHERE status = 'IN_PROGRESS' AND claimedAt < :cutoff") - int recoverStaleInProgress(@Bind("cutoff") java.sql.Timestamp cutoff); - - @SqlUpdate( - "UPDATE search_index_retry_queue SET status = :status, failureReason = :failureReason, " - + "retryCount = retryCount + 1, claimedAt = NULL " - + "WHERE entityId = :entityId AND entityFqn = :entityFqn") - int updateFailureAndRetryCount( - @Bind("entityId") String entityId, - @Bind("entityFqn") String entityFqn, - @Bind("failureReason") String failureReason, - @Bind("status") String status); - - default List claimPending(int batchSize) { - int fetchSize = Math.max(batchSize * 5, batchSize); - List candidates = - new ArrayList<>( - findByStatuses(List.of("PENDING", "PENDING_RETRY_1", "PENDING_RETRY_2"), fetchSize)); - // Shuffle so concurrent worker threads attempt different rows first, - // reducing wasted optimistic-lock failures on the same candidates. - Collections.shuffle(candidates); - List claimed = new ArrayList<>(); - for (SearchIndexRetryRecord candidate : candidates) { - if (claimed.size() >= batchSize) { - break; - } - int updated = - claimRecord(candidate.getEntityId(), candidate.getEntityFqn(), candidate.getStatus()); - if (updated == 1) { - claimed.add(candidate); - } - } - return claimed; - } - - @SqlQuery( - "SELECT entityId, entityFqn, failureReason, status, entityType, retryCount, claimedAt " - + "FROM search_index_retry_queue ORDER BY retryCount DESC, claimedAt DESC " - + "LIMIT :limit OFFSET :offset") - @RegisterRowMapper(SearchIndexRetryRecordMapper.class) - List listAll(@Bind("limit") int limit, @Bind("offset") int offset); - - @SqlQuery("SELECT COUNT(*) FROM search_index_retry_queue") - int countAll(); - - class SearchIndexRetryRecordMapper implements RowMapper { - @Override - public SearchIndexRetryRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new SearchIndexRetryRecord( - rs.getString("entityId"), - rs.getString("entityFqn"), - rs.getString("failureReason"), - rs.getString("status"), - rs.getString("entityType"), - rs.getInt("retryCount"), - rs.getTimestamp("claimedAt")); - } - } - } - - /** DAO for search index per-server stats in distributed mode */ - interface SearchIndexServerStatsDAO { - - record ServerStatsRecord( - String id, - String jobId, - String serverId, - String entityType, - long readerSuccess, - long readerFailed, - long readerWarnings, - long sinkSuccess, - long sinkFailed, - long processSuccess, - long processFailed, - long vectorSuccess, - long vectorFailed, - long readerTimeMs, - long processTimeMs, - long sinkTimeMs, - long vectorTimeMs, - int partitionsCompleted, - int partitionsFailed, - long lastUpdatedAt) {} - - record AggregatedServerStats( - long readerSuccess, - long readerFailed, - long readerWarnings, - long sinkSuccess, - long sinkFailed, - long processSuccess, - long processFailed, - long vectorSuccess, - long vectorFailed, - long readerTimeMs, - long processTimeMs, - long sinkTimeMs, - long vectorTimeMs, - int partitionsCompleted, - int partitionsFailed) {} - - record EntityStats( - String entityType, - long readerSuccess, - long readerFailed, - long readerWarnings, - long sinkSuccess, - long sinkFailed, - long processSuccess, - long processFailed, - long vectorSuccess, - long vectorFailed, - long readerTimeMs, - long processTimeMs, - long sinkTimeMs, - long vectorTimeMs) {} - - /** - * Increment stats using delta values. This is the primary method for updating stats - - * it adds the delta values to existing values, creating the row if it doesn't exist. - */ - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_server_stats (id, jobId, serverId, entityType, " - + "readerSuccess, readerFailed, readerWarnings, sinkSuccess, sinkFailed, " - + "processSuccess, processFailed, vectorSuccess, vectorFailed, " - + "readerTimeMs, processTimeMs, sinkTimeMs, vectorTimeMs, " - + "partitionsCompleted, partitionsFailed, lastUpdatedAt) " - + "VALUES (:id, :jobId, :serverId, :entityType, " - + ":readerSuccess, :readerFailed, :readerWarnings, :sinkSuccess, :sinkFailed, " - + ":processSuccess, :processFailed, :vectorSuccess, :vectorFailed, " - + ":readerTimeMs, :processTimeMs, :sinkTimeMs, :vectorTimeMs, " - + ":partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " - + "ON DUPLICATE KEY UPDATE " - + "readerSuccess = readerSuccess + VALUES(readerSuccess), " - + "readerFailed = readerFailed + VALUES(readerFailed), " - + "readerWarnings = readerWarnings + VALUES(readerWarnings), " - + "sinkSuccess = sinkSuccess + VALUES(sinkSuccess), " - + "sinkFailed = sinkFailed + VALUES(sinkFailed), " - + "processSuccess = processSuccess + VALUES(processSuccess), " - + "processFailed = processFailed + VALUES(processFailed), " - + "vectorSuccess = vectorSuccess + VALUES(vectorSuccess), " - + "vectorFailed = vectorFailed + VALUES(vectorFailed), " - + "readerTimeMs = readerTimeMs + VALUES(readerTimeMs), " - + "processTimeMs = processTimeMs + VALUES(processTimeMs), " - + "sinkTimeMs = sinkTimeMs + VALUES(sinkTimeMs), " - + "vectorTimeMs = vectorTimeMs + VALUES(vectorTimeMs), " - + "partitionsCompleted = partitionsCompleted + VALUES(partitionsCompleted), " - + "partitionsFailed = partitionsFailed + VALUES(partitionsFailed), " - + "lastUpdatedAt = VALUES(lastUpdatedAt)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_server_stats (id, jobId, serverId, entityType, " - + "readerSuccess, readerFailed, readerWarnings, sinkSuccess, sinkFailed, " - + "processSuccess, processFailed, vectorSuccess, vectorFailed, " - + "readerTimeMs, processTimeMs, sinkTimeMs, vectorTimeMs, " - + "partitionsCompleted, partitionsFailed, lastUpdatedAt) " - + "VALUES (:id, :jobId, :serverId, :entityType, " - + ":readerSuccess, :readerFailed, :readerWarnings, :sinkSuccess, :sinkFailed, " - + ":processSuccess, :processFailed, :vectorSuccess, :vectorFailed, " - + ":readerTimeMs, :processTimeMs, :sinkTimeMs, :vectorTimeMs, " - + ":partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " - + "ON CONFLICT (jobId, serverId, entityType) DO UPDATE SET " - + "readerSuccess = search_index_server_stats.readerSuccess + EXCLUDED.readerSuccess, " - + "readerFailed = search_index_server_stats.readerFailed + EXCLUDED.readerFailed, " - + "readerWarnings = search_index_server_stats.readerWarnings + EXCLUDED.readerWarnings, " - + "sinkSuccess = search_index_server_stats.sinkSuccess + EXCLUDED.sinkSuccess, " - + "sinkFailed = search_index_server_stats.sinkFailed + EXCLUDED.sinkFailed, " - + "processSuccess = search_index_server_stats.processSuccess + EXCLUDED.processSuccess, " - + "processFailed = search_index_server_stats.processFailed + EXCLUDED.processFailed, " - + "vectorSuccess = search_index_server_stats.vectorSuccess + EXCLUDED.vectorSuccess, " - + "vectorFailed = search_index_server_stats.vectorFailed + EXCLUDED.vectorFailed, " - + "readerTimeMs = search_index_server_stats.readerTimeMs + EXCLUDED.readerTimeMs, " - + "processTimeMs = search_index_server_stats.processTimeMs + EXCLUDED.processTimeMs, " - + "sinkTimeMs = search_index_server_stats.sinkTimeMs + EXCLUDED.sinkTimeMs, " - + "vectorTimeMs = search_index_server_stats.vectorTimeMs + EXCLUDED.vectorTimeMs, " - + "partitionsCompleted = search_index_server_stats.partitionsCompleted + EXCLUDED.partitionsCompleted, " - + "partitionsFailed = search_index_server_stats.partitionsFailed + EXCLUDED.partitionsFailed, " - + "lastUpdatedAt = EXCLUDED.lastUpdatedAt", - connectionType = POSTGRES) - void incrementStats( - @Bind("id") String id, - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("entityType") String entityType, - @Bind("readerSuccess") long readerSuccess, - @Bind("readerFailed") long readerFailed, - @Bind("readerWarnings") long readerWarnings, - @Bind("sinkSuccess") long sinkSuccess, - @Bind("sinkFailed") long sinkFailed, - @Bind("processSuccess") long processSuccess, - @Bind("processFailed") long processFailed, - @Bind("vectorSuccess") long vectorSuccess, - @Bind("vectorFailed") long vectorFailed, - @Bind("readerTimeMs") long readerTimeMs, - @Bind("processTimeMs") long processTimeMs, - @Bind("sinkTimeMs") long sinkTimeMs, - @Bind("vectorTimeMs") long vectorTimeMs, - @Bind("partitionsCompleted") int partitionsCompleted, - @Bind("partitionsFailed") int partitionsFailed, - @Bind("lastUpdatedAt") long lastUpdatedAt); - - /** - * Replace stats with absolute values. Used by distributed coordinator to persist - * aggregate stats for the server. - */ - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_server_stats (id, jobId, serverId, entityType, " - + "readerSuccess, readerFailed, readerWarnings, sinkSuccess, sinkFailed, " - + "processSuccess, processFailed, vectorSuccess, vectorFailed, " - + "readerTimeMs, processTimeMs, sinkTimeMs, vectorTimeMs, " - + "partitionsCompleted, partitionsFailed, lastUpdatedAt) " - + "VALUES (:id, :jobId, :serverId, :entityType, " - + ":readerSuccess, :readerFailed, :readerWarnings, :sinkSuccess, :sinkFailed, " - + ":processSuccess, :processFailed, :vectorSuccess, :vectorFailed, " - + ":readerTimeMs, :processTimeMs, :sinkTimeMs, :vectorTimeMs, " - + ":partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " - + "ON DUPLICATE KEY UPDATE " - + "readerSuccess = VALUES(readerSuccess), " - + "readerFailed = VALUES(readerFailed), " - + "readerWarnings = VALUES(readerWarnings), " - + "sinkSuccess = VALUES(sinkSuccess), " - + "sinkFailed = VALUES(sinkFailed), " - + "processSuccess = VALUES(processSuccess), " - + "processFailed = VALUES(processFailed), " - + "vectorSuccess = VALUES(vectorSuccess), " - + "vectorFailed = VALUES(vectorFailed), " - + "readerTimeMs = VALUES(readerTimeMs), " - + "processTimeMs = VALUES(processTimeMs), " - + "sinkTimeMs = VALUES(sinkTimeMs), " - + "vectorTimeMs = VALUES(vectorTimeMs), " - + "partitionsCompleted = VALUES(partitionsCompleted), " - + "partitionsFailed = VALUES(partitionsFailed), " - + "lastUpdatedAt = VALUES(lastUpdatedAt)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO search_index_server_stats (id, jobId, serverId, entityType, " - + "readerSuccess, readerFailed, readerWarnings, sinkSuccess, sinkFailed, " - + "processSuccess, processFailed, vectorSuccess, vectorFailed, " - + "readerTimeMs, processTimeMs, sinkTimeMs, vectorTimeMs, " - + "partitionsCompleted, partitionsFailed, lastUpdatedAt) " - + "VALUES (:id, :jobId, :serverId, :entityType, " - + ":readerSuccess, :readerFailed, :readerWarnings, :sinkSuccess, :sinkFailed, " - + ":processSuccess, :processFailed, :vectorSuccess, :vectorFailed, " - + ":readerTimeMs, :processTimeMs, :sinkTimeMs, :vectorTimeMs, " - + ":partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " - + "ON CONFLICT (jobId, serverId, entityType) DO UPDATE SET " - + "readerSuccess = EXCLUDED.readerSuccess, " - + "readerFailed = EXCLUDED.readerFailed, " - + "readerWarnings = EXCLUDED.readerWarnings, " - + "sinkSuccess = EXCLUDED.sinkSuccess, " - + "sinkFailed = EXCLUDED.sinkFailed, " - + "processSuccess = EXCLUDED.processSuccess, " - + "processFailed = EXCLUDED.processFailed, " - + "vectorSuccess = EXCLUDED.vectorSuccess, " - + "vectorFailed = EXCLUDED.vectorFailed, " - + "readerTimeMs = EXCLUDED.readerTimeMs, " - + "processTimeMs = EXCLUDED.processTimeMs, " - + "sinkTimeMs = EXCLUDED.sinkTimeMs, " - + "vectorTimeMs = EXCLUDED.vectorTimeMs, " - + "partitionsCompleted = EXCLUDED.partitionsCompleted, " - + "partitionsFailed = EXCLUDED.partitionsFailed, " - + "lastUpdatedAt = EXCLUDED.lastUpdatedAt", - connectionType = POSTGRES) - void replaceStats( - @Bind("id") String id, - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("entityType") String entityType, - @Bind("readerSuccess") long readerSuccess, - @Bind("readerFailed") long readerFailed, - @Bind("readerWarnings") long readerWarnings, - @Bind("sinkSuccess") long sinkSuccess, - @Bind("sinkFailed") long sinkFailed, - @Bind("processSuccess") long processSuccess, - @Bind("processFailed") long processFailed, - @Bind("vectorSuccess") long vectorSuccess, - @Bind("vectorFailed") long vectorFailed, - @Bind("readerTimeMs") long readerTimeMs, - @Bind("processTimeMs") long processTimeMs, - @Bind("sinkTimeMs") long sinkTimeMs, - @Bind("vectorTimeMs") long vectorTimeMs, - @Bind("partitionsCompleted") int partitionsCompleted, - @Bind("partitionsFailed") int partitionsFailed, - @Bind("lastUpdatedAt") long lastUpdatedAt); - - @SqlQuery("SELECT * FROM search_index_server_stats WHERE jobId = :jobId") - @RegisterRowMapper(ServerStatsMapper.class) - List findByJobId(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT * FROM search_index_server_stats WHERE jobId = :jobId AND serverId = :serverId AND entityType = :entityType") - @RegisterRowMapper(ServerStatsMapper.class) - ServerStatsRecord findByJobIdServerIdEntityType( - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("entityType") String entityType); - - /** Get aggregated stats across all servers and entity types for a job */ - @SqlQuery( - "SELECT " - + "COALESCE(SUM(readerSuccess), 0) as readerSuccess, " - + "COALESCE(SUM(readerFailed), 0) as readerFailed, " - + "COALESCE(SUM(readerWarnings), 0) as readerWarnings, " - + "COALESCE(SUM(sinkSuccess), 0) as sinkSuccess, " - + "COALESCE(SUM(sinkFailed), 0) as sinkFailed, " - + "COALESCE(SUM(processSuccess), 0) as processSuccess, " - + "COALESCE(SUM(processFailed), 0) as processFailed, " - + "COALESCE(SUM(vectorSuccess), 0) as vectorSuccess, " - + "COALESCE(SUM(vectorFailed), 0) as vectorFailed, " - + "COALESCE(SUM(readerTimeMs), 0) as readerTimeMs, " - + "COALESCE(SUM(processTimeMs), 0) as processTimeMs, " - + "COALESCE(SUM(sinkTimeMs), 0) as sinkTimeMs, " - + "COALESCE(SUM(vectorTimeMs), 0) as vectorTimeMs, " - + "COALESCE(SUM(partitionsCompleted), 0) as partitionsCompleted, " - + "COALESCE(SUM(partitionsFailed), 0) as partitionsFailed " - + "FROM search_index_server_stats WHERE jobId = :jobId") - @RegisterRowMapper(AggregatedServerStatsMapper.class) - AggregatedServerStats getAggregatedStats(@Bind("jobId") String jobId); - - /** Get stats grouped by entity type for a job */ - @SqlQuery( - "SELECT entityType, " - + "COALESCE(SUM(readerSuccess), 0) as readerSuccess, " - + "COALESCE(SUM(readerFailed), 0) as readerFailed, " - + "COALESCE(SUM(readerWarnings), 0) as readerWarnings, " - + "COALESCE(SUM(sinkSuccess), 0) as sinkSuccess, " - + "COALESCE(SUM(sinkFailed), 0) as sinkFailed, " - + "COALESCE(SUM(processSuccess), 0) as processSuccess, " - + "COALESCE(SUM(processFailed), 0) as processFailed, " - + "COALESCE(SUM(vectorSuccess), 0) as vectorSuccess, " - + "COALESCE(SUM(vectorFailed), 0) as vectorFailed, " - + "COALESCE(SUM(readerTimeMs), 0) as readerTimeMs, " - + "COALESCE(SUM(processTimeMs), 0) as processTimeMs, " - + "COALESCE(SUM(sinkTimeMs), 0) as sinkTimeMs, " - + "COALESCE(SUM(vectorTimeMs), 0) as vectorTimeMs " - + "FROM search_index_server_stats WHERE jobId = :jobId " - + "GROUP BY entityType") - @RegisterRowMapper(EntityStatsMapper.class) - List getStatsByEntityType(@Bind("jobId") String jobId); - - /** - * Per-server timing breakdown. Sums every counter and timing column for each serverId, - * letting the UI show "is one node dragging the cluster" for distributed runs. - */ - record ServerTimingStats( - String serverId, - long readerSuccess, - long sinkSuccess, - long processSuccess, - long vectorSuccess, - long readerTimeMs, - long processTimeMs, - long sinkTimeMs, - long vectorTimeMs) {} - - @SqlQuery( - "SELECT serverId, " - + "COALESCE(SUM(readerSuccess), 0) as readerSuccess, " - + "COALESCE(SUM(sinkSuccess), 0) as sinkSuccess, " - + "COALESCE(SUM(processSuccess), 0) as processSuccess, " - + "COALESCE(SUM(vectorSuccess), 0) as vectorSuccess, " - + "COALESCE(SUM(readerTimeMs), 0) as readerTimeMs, " - + "COALESCE(SUM(processTimeMs), 0) as processTimeMs, " - + "COALESCE(SUM(sinkTimeMs), 0) as sinkTimeMs, " - + "COALESCE(SUM(vectorTimeMs), 0) as vectorTimeMs " - + "FROM search_index_server_stats WHERE jobId = :jobId " - + "GROUP BY serverId") - @RegisterRowMapper(ServerTimingStatsMapper.class) - List getStatsByServer(@Bind("jobId") String jobId); - - class ServerTimingStatsMapper implements RowMapper { - @Override - public ServerTimingStats map(ResultSet rs, StatementContext ctx) throws SQLException { - return new ServerTimingStats( - rs.getString("serverId"), - rs.getLong("readerSuccess"), - rs.getLong("sinkSuccess"), - rs.getLong("processSuccess"), - rs.getLong("vectorSuccess"), - rs.getLong("readerTimeMs"), - rs.getLong("processTimeMs"), - rs.getLong("sinkTimeMs"), - rs.getLong("vectorTimeMs")); - } - } - - @SqlUpdate("DELETE FROM search_index_server_stats WHERE jobId = :jobId") - void deleteByJobId(@Bind("jobId") String jobId); - - @SqlUpdate("DELETE FROM search_index_server_stats") - void deleteAll(); - - class ServerStatsMapper implements RowMapper { - @Override - public ServerStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new ServerStatsRecord( - rs.getString("id"), - rs.getString("jobId"), - rs.getString("serverId"), - rs.getString("entityType"), - rs.getLong("readerSuccess"), - rs.getLong("readerFailed"), - rs.getLong("readerWarnings"), - rs.getLong("sinkSuccess"), - rs.getLong("sinkFailed"), - rs.getLong("processSuccess"), - rs.getLong("processFailed"), - rs.getLong("vectorSuccess"), - rs.getLong("vectorFailed"), - rs.getLong("readerTimeMs"), - rs.getLong("processTimeMs"), - rs.getLong("sinkTimeMs"), - rs.getLong("vectorTimeMs"), - rs.getInt("partitionsCompleted"), - rs.getInt("partitionsFailed"), - rs.getLong("lastUpdatedAt")); - } - } - - class AggregatedServerStatsMapper implements RowMapper { - @Override - public AggregatedServerStats map(ResultSet rs, StatementContext ctx) throws SQLException { - return new AggregatedServerStats( - rs.getLong("readerSuccess"), - rs.getLong("readerFailed"), - rs.getLong("readerWarnings"), - rs.getLong("sinkSuccess"), - rs.getLong("sinkFailed"), - rs.getLong("processSuccess"), - rs.getLong("processFailed"), - rs.getLong("vectorSuccess"), - rs.getLong("vectorFailed"), - rs.getLong("readerTimeMs"), - rs.getLong("processTimeMs"), - rs.getLong("sinkTimeMs"), - rs.getLong("vectorTimeMs"), - rs.getInt("partitionsCompleted"), - rs.getInt("partitionsFailed")); - } - } - - class EntityStatsMapper implements RowMapper { - @Override - public EntityStats map(ResultSet rs, StatementContext ctx) throws SQLException { - return new EntityStats( - rs.getString("entityType"), - rs.getLong("readerSuccess"), - rs.getLong("readerFailed"), - rs.getLong("readerWarnings"), - rs.getLong("sinkSuccess"), - rs.getLong("sinkFailed"), - rs.getLong("processSuccess"), - rs.getLong("processFailed"), - rs.getLong("vectorSuccess"), - rs.getLong("vectorFailed"), - rs.getLong("readerTimeMs"), - rs.getLong("processTimeMs"), - rs.getLong("sinkTimeMs"), - rs.getLong("vectorTimeMs")); - } - } - } - - @Builder - record ActivityStreamRow( - String id, - String eventType, - String entityType, - String entityId, - String entityFqnHash, - String about, - String aboutFqnHash, - String actorId, - String actorName, - Long timestamp, - String summary, - String fieldName, - String oldValue, - String newValue, - String domains, - String json) {} - - interface ActivityStreamDAO { - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO activity_stream(id, eventType, entityType, entityId, entityFqnHash, " - + "about, aboutFqnHash, actorId, actorName, timestamp, summary, fieldName, oldValue, newValue, domains, json) " - + "VALUES (:id, :eventType, :entityType, :entityId, :entityFqnHash, " - + ":about, :aboutFqnHash, :actorId, :actorName, :timestamp, :summary, :fieldName, :oldValue, :newValue, :domains, :json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO activity_stream(id, eventtype, entitytype, entityid, entityfqnhash, " - + "about, aboutfqnhash, actorid, actorname, timestamp, summary, fieldname, oldvalue, newvalue, domains, json) " - + "VALUES (:id, :eventType, :entityType, :entityId, :entityFqnHash, " - + ":about, :aboutFqnHash, :actorId, :actorName, :timestamp, :summary, :fieldName, :oldValue, :newValue, :domains::jsonb, :json::jsonb)", - connectionType = POSTGRES) - void insert( - @Bind("id") String id, - @Bind("eventType") String eventType, - @Bind("entityType") String entityType, - @Bind("entityId") String entityId, - @Bind("entityFqnHash") String entityFqnHash, - @Bind("about") String about, - @Bind("aboutFqnHash") String aboutFqnHash, - @Bind("actorId") String actorId, - @Bind("actorName") String actorName, - @Bind("timestamp") long timestamp, - @Bind("summary") String summary, - @Bind("fieldName") String fieldName, - @Bind("oldValue") String oldValue, - @Bind("newValue") String newValue, - @Bind("domains") String domains, - @Bind("json") String json); - - // Batch insert for activity events - one round-trip per change event instead of one per row - @Transaction - @ConnectionAwareSqlBatch( - value = - "INSERT INTO activity_stream(id, eventType, entityType, entityId, entityFqnHash, " - + "about, aboutFqnHash, actorId, actorName, timestamp, summary, fieldName, oldValue, newValue, domains, json) " - + "VALUES (:id, :eventType, :entityType, :entityId, :entityFqnHash, " - + ":about, :aboutFqnHash, :actorId, :actorName, :timestamp, :summary, :fieldName, :oldValue, :newValue, :domains, :json)", - connectionType = MYSQL) - @ConnectionAwareSqlBatch( - value = - "INSERT INTO activity_stream(id, eventtype, entitytype, entityid, entityfqnhash, " - + "about, aboutfqnhash, actorid, actorname, timestamp, summary, fieldname, oldvalue, newvalue, domains, json) " - + "VALUES (:id, :eventType, :entityType, :entityId, :entityFqnHash, " - + ":about, :aboutFqnHash, :actorId, :actorName, :timestamp, :summary, :fieldName, :oldValue, :newValue, :domains::jsonb, :json::jsonb)", - connectionType = POSTGRES) - void insertBatch(@BindMethods List rows); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE timestamp >= :after " - + "ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE timestamp >= :after " - + "ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List list(@Bind("after") long after, @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entityType = :entityType AND entityId = :entityId " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entitytype = :entityType AND entityid = :entityId " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByEntity( - @Bind("entityType") String entityType, - @Bind("entityId") String entityId, - @Bind("after") long after, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entityType = :entityType " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entitytype = :entityType " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByEntityType( - @Bind("entityType") String entityType, @Bind("after") long after, @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE actorId = :actorId " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE actorid = :actorId " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByActor( - @Bind("actorId") String actorId, @Bind("after") long after, @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE actorId = :actorId " - + "AND JSON_OVERLAPS(domains, :domainJson) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE actorid = :actorId " - + "AND EXISTS (" - + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " - + "WHERE domain_id IN ()) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByActorAndDomains( - @Bind("actorId") String actorId, - @Bind("domainJson") String domainJson, - @BindList("domainIds") List domainIds, - @Bind("after") long after, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE JSON_OVERLAPS(domains, :domainJson) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE EXISTS (" - + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " - + "WHERE domain_id IN ()) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByDomains( - @Bind("domainJson") String domainJson, - @BindList("domainIds") List domainIds, - @Bind("after") long after, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entityType = :entityType AND entityId = :entityId " - + "AND JSON_OVERLAPS(domains, :domainJson) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entitytype = :entityType AND entityid = :entityId " - + "AND EXISTS (" - + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " - + "WHERE domain_id IN ()) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByEntityAndDomains( - @Bind("entityType") String entityType, - @Bind("entityId") String entityId, - @Bind("domainJson") String domainJson, - @BindList("domainIds") List domainIds, - @Bind("after") long after, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entityType = :entityType " - + "AND JSON_OVERLAPS(domains, :domainJson) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entitytype = :entityType " - + "AND EXISTS (" - + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " - + "WHERE domain_id IN ()) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByEntityTypeAndDomains( - @Bind("entityType") String entityType, - @Bind("domainJson") String domainJson, - @BindList("domainIds") List domainIds, - @Bind("after") long after, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = "SELECT count(*) FROM activity_stream WHERE timestamp >= :after", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT count(*) FROM activity_stream WHERE timestamp >= :after", - connectionType = POSTGRES) - int count(@Bind("after") long after); - - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM activity_stream WHERE JSON_OVERLAPS(domains, :domainJson) " - + "AND timestamp >= :after", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM activity_stream WHERE EXISTS (" - + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " - + "WHERE domain_id IN ()) " - + "AND timestamp >= :after", - connectionType = POSTGRES) - int countByDomains( - @Bind("domainJson") String domainJson, - @BindList("domainIds") List domainIds, - @Bind("after") long after); - - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM activity_stream WHERE entityType = :entityType AND entityId = :entityId " - + "AND timestamp >= :after", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM activity_stream WHERE entitytype = :entityType AND entityid = :entityId " - + "AND timestamp >= :after", - connectionType = POSTGRES) - int countByEntity( - @Bind("entityType") String entityType, - @Bind("entityId") String entityId, - @Bind("after") long after); - - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM activity_stream WHERE entityType = :entityType AND entityId = :entityId " - + "AND JSON_OVERLAPS(domains, :domainJson) " - + "AND timestamp >= :after", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT count(*) FROM activity_stream WHERE entitytype = :entityType AND entityid = :entityId " - + "AND EXISTS (" - + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " - + "WHERE domain_id IN ()) " - + "AND timestamp >= :after", - connectionType = POSTGRES) - int countByEntityAndDomains( - @Bind("entityType") String entityType, - @Bind("entityId") String entityId, - @Bind("domainJson") String domainJson, - @BindList("domainIds") List domainIds, - @Bind("after") long after); - - @SqlUpdate("DELETE FROM activity_stream WHERE timestamp < :cutoff") - int deleteOlderThan(@Bind("cutoff") long cutoffTimestamp); - - @SqlQuery("SELECT json FROM activity_stream WHERE id = :id") - String findById(@Bind("id") String id); - - @ConnectionAwareSqlUpdate( - value = "UPDATE activity_stream SET json = :json WHERE id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE activity_stream SET json = :json::jsonb WHERE id = :id", - connectionType = POSTGRES) - void updateJson(@Bind("id") String id, @Bind("json") String json); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entityId IN (" - + "SELECT toId FROM entity_relationship WHERE relation = 8 " - + "AND ((fromEntity = 'user' AND fromId = :userId) " - + "OR (fromEntity = 'team' AND fromId IN ()))) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entityid IN (" - + "SELECT toid FROM entity_relationship WHERE relation = 8 " - + "AND ((fromentity = 'user' AND fromid = :userId) " - + "OR (fromentity = 'team' AND fromid IN ()))) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByOwners( - @Bind("userId") String userId, - @BindList("teamIds") List teamIds, - @Bind("after") long after, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entityId IN (" - + "SELECT toId FROM entity_relationship WHERE relation = 8 " - + "AND ((fromEntity = 'user' AND fromId = :userId) " - + "OR (fromEntity = 'team' AND fromId IN ()))) " - + "AND JSON_OVERLAPS(domains, :domainJson) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE entityid IN (" - + "SELECT toid FROM entity_relationship WHERE relation = 8 " - + "AND ((fromentity = 'user' AND fromid = :userId) " - + "OR (fromentity = 'team' AND fromid IN ()))) " - + "AND EXISTS (" - + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " - + "WHERE domain_id IN ()) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByOwnersAndDomains( - @Bind("userId") String userId, - @BindList("teamIds") List teamIds, - @Bind("domainJson") String domainJson, - @BindList("domainIds") List domainIds, - @Bind("after") long after, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE aboutFqnHash = :aboutFqnHash " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE aboutfqnhash = :aboutFqnHash " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByAbout( - @Bind("aboutFqnHash") String aboutFqnHash, - @Bind("after") long after, - @Bind("limit") int limit); - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE aboutFqnHash = :aboutFqnHash " - + "AND JSON_OVERLAPS(domains, :domainJson) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM activity_stream WHERE aboutfqnhash = :aboutFqnHash " - + "AND EXISTS (" - + "SELECT 1 FROM jsonb_array_elements_text(domains) AS domain_id " - + "WHERE domain_id IN ()) " - + "AND timestamp >= :after ORDER BY timestamp DESC, id DESC LIMIT :limit", - connectionType = POSTGRES) - List listByAboutAndDomains( - @Bind("aboutFqnHash") String aboutFqnHash, - @Bind("domainJson") String domainJson, - @BindList("domainIds") List domainIds, - @Bind("after") long after, - @Bind("limit") int limit); - } - - interface ActivityStreamConfigDAO { - @ConnectionAwareSqlUpdate( - value = "INSERT INTO activity_stream_config(id, json) VALUES (:id, :json)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "INSERT INTO activity_stream_config(id, json) VALUES (:id, :json::jsonb)", - connectionType = POSTGRES) - void insert(@Bind("id") String id, @Bind("json") String json); - - @ConnectionAwareSqlUpdate( - value = "UPDATE activity_stream_config SET json = :json WHERE id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = "UPDATE activity_stream_config SET json = :json::jsonb WHERE id = :id", - connectionType = POSTGRES) - void update(@Bind("id") String id, @Bind("json") String json); - - @SqlQuery("SELECT json FROM activity_stream_config WHERE id = :id") - String findById(@Bind("id") String id); - - @ConnectionAwareSqlQuery( - value = "SELECT json FROM activity_stream_config WHERE domainId = :domainId", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = "SELECT json FROM activity_stream_config WHERE domainid = :domainId", - connectionType = POSTGRES) - String findByDomainId(@Bind("domainId") String domainId); - - @SqlQuery("SELECT json FROM activity_stream_config WHERE scope = 'global' LIMIT 1") - String findGlobalConfig(); - - @SqlQuery("SELECT json FROM activity_stream_config") - List listAll(); - - @SqlUpdate("DELETE FROM activity_stream_config WHERE id = :id") - void delete(@Bind("id") String id); - } - - /** DAO for distributed RDF index jobs. */ - interface RdfIndexJobDAO { - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO rdf_index_job (id, status, jobConfiguration, totalRecords, processedRecords, " - + "successRecords, failedRecords, stats, createdBy, createdAt, updatedAt) " - + "VALUES (:id, :status, :jobConfiguration, :totalRecords, :processedRecords, " - + ":successRecords, :failedRecords, :stats, :createdBy, :createdAt, :updatedAt)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO rdf_index_job (id, status, jobConfiguration, totalRecords, processedRecords, " - + "successRecords, failedRecords, stats, createdBy, createdAt, updatedAt) " - + "VALUES (:id, :status, :jobConfiguration::jsonb, :totalRecords, :processedRecords, " - + ":successRecords, :failedRecords, :stats::jsonb, :createdBy, :createdAt, :updatedAt)", - connectionType = POSTGRES) - void insert( - @Bind("id") String id, - @Bind("status") String status, - @Bind("jobConfiguration") String jobConfiguration, - @Bind("totalRecords") long totalRecords, - @Bind("processedRecords") long processedRecords, - @Bind("successRecords") long successRecords, - @Bind("failedRecords") long failedRecords, - @Bind("stats") String stats, - @Bind("createdBy") String createdBy, - @Bind("createdAt") long createdAt, - @Bind("updatedAt") long updatedAt); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE rdf_index_job SET status = :status, processedRecords = :processedRecords, " - + "successRecords = :successRecords, failedRecords = :failedRecords, stats = :stats, " - + "startedAt = :startedAt, completedAt = :completedAt, updatedAt = :updatedAt, " - + "errorMessage = :errorMessage WHERE id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE rdf_index_job SET status = :status, processedRecords = :processedRecords, " - + "successRecords = :successRecords, failedRecords = :failedRecords, stats = :stats::jsonb, " - + "startedAt = :startedAt, completedAt = :completedAt, updatedAt = :updatedAt, " - + "errorMessage = :errorMessage WHERE id = :id", - connectionType = POSTGRES) - void update( - @Bind("id") String id, - @Bind("status") String status, - @Bind("processedRecords") long processedRecords, - @Bind("successRecords") long successRecords, - @Bind("failedRecords") long failedRecords, - @Bind("stats") String stats, - @Bind("startedAt") Long startedAt, - @Bind("completedAt") Long completedAt, - @Bind("updatedAt") long updatedAt, - @Bind("errorMessage") String errorMessage); - - @SqlUpdate("UPDATE rdf_index_job SET updatedAt = :updatedAt WHERE id = :id") - void touchJob(@Bind("id") String id, @Bind("updatedAt") long updatedAt); - - @SqlQuery("SELECT * FROM rdf_index_job WHERE id = :id") - @RegisterRowMapper(RdfIndexJobMapper.class) - RdfIndexJobRecord findById(@Bind("id") String id); - - @SqlQuery("SELECT * FROM rdf_index_job WHERE status IN () ORDER BY createdAt DESC") - @RegisterRowMapper(RdfIndexJobMapper.class) - List findByStatuses(@BindList("statuses") List statuses); - - @SqlQuery( - "SELECT * FROM rdf_index_job WHERE status IN () ORDER BY createdAt DESC LIMIT :limit") - @RegisterRowMapper(RdfIndexJobMapper.class) - List findByStatusesWithLimit( - @BindList("statuses") List statuses, @Bind("limit") int limit); - - @SqlQuery("SELECT id FROM rdf_index_job WHERE status IN ('READY', 'RUNNING', 'STOPPING')") - List getRunningJobIds(); - - @SqlUpdate("DELETE FROM rdf_index_job") - void deleteAll(); - - class RdfIndexJobMapper implements RowMapper { - @Override - public RdfIndexJobRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new RdfIndexJobRecord( - rs.getString("id"), - rs.getString("status"), - rs.getString("jobConfiguration"), - rs.getLong("totalRecords"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getString("stats"), - rs.getString("createdBy"), - rs.getLong("createdAt"), - (Long) rs.getObject("startedAt"), - (Long) rs.getObject("completedAt"), - rs.getLong("updatedAt"), - rs.getString("errorMessage")); - } - } - - record RdfIndexJobRecord( - String id, - String status, - String jobConfiguration, - long totalRecords, - long processedRecords, - long successRecords, - long failedRecords, - String stats, - String createdBy, - long createdAt, - Long startedAt, - Long completedAt, - long updatedAt, - String errorMessage) {} - } - - /** DAO for distributed RDF partitions. */ - interface RdfIndexPartitionDAO { - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO rdf_index_partition (id, jobId, entityType, partitionIndex, rangeStart, rangeEnd, " - + "estimatedCount, workUnits, priority, status, processingCursor, claimableAt) " - + "VALUES (:id, :jobId, :entityType, :partitionIndex, :rangeStart, :rangeEnd, " - + ":estimatedCount, :workUnits, :priority, :status, :cursor, :claimableAt)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO rdf_index_partition (id, jobId, entityType, partitionIndex, rangeStart, rangeEnd, " - + "estimatedCount, workUnits, priority, status, processingCursor, claimableAt) " - + "VALUES (:id, :jobId, :entityType, :partitionIndex, :rangeStart, :rangeEnd, " - + ":estimatedCount, :workUnits, :priority, :status, :cursor, :claimableAt)", - connectionType = POSTGRES) - void insert( - @Bind("id") String id, - @Bind("jobId") String jobId, - @Bind("entityType") String entityType, - @Bind("partitionIndex") int partitionIndex, - @Bind("rangeStart") long rangeStart, - @Bind("rangeEnd") long rangeEnd, - @Bind("estimatedCount") long estimatedCount, - @Bind("workUnits") long workUnits, - @Bind("priority") int priority, - @Bind("status") String status, - @Bind("cursor") long cursor, - @Bind("claimableAt") long claimableAt); - - @SqlUpdate( - "UPDATE rdf_index_partition SET status = :status, processingCursor = :cursor, " - + "processedCount = :processedCount, successCount = :successCount, failedCount = :failedCount, " - + "assignedServer = :assignedServer, claimedAt = :claimedAt, startedAt = :startedAt, " - + "completedAt = :completedAt, lastUpdateAt = :lastUpdateAt, lastError = :lastError, " - + "retryCount = :retryCount WHERE id = :id") - void update( - @Bind("id") String id, - @Bind("status") String status, - @Bind("cursor") long cursor, - @Bind("processedCount") long processedCount, - @Bind("successCount") long successCount, - @Bind("failedCount") long failedCount, - @Bind("assignedServer") String assignedServer, - @Bind("claimedAt") Long claimedAt, - @Bind("startedAt") Long startedAt, - @Bind("completedAt") Long completedAt, - @Bind("lastUpdateAt") Long lastUpdateAt, - @Bind("lastError") String lastError, - @Bind("retryCount") int retryCount); - - @SqlUpdate( - "UPDATE rdf_index_partition SET processingCursor = :cursor, processedCount = :processedCount, " - + "successCount = :successCount, failedCount = :failedCount, lastUpdateAt = :lastUpdateAt " - + "WHERE id = :id") - void updateProgress( - @Bind("id") String id, - @Bind("cursor") long cursor, - @Bind("processedCount") long processedCount, - @Bind("successCount") long successCount, - @Bind("failedCount") long failedCount, - @Bind("lastUpdateAt") long lastUpdateAt); - - @SqlUpdate("UPDATE rdf_index_partition SET lastUpdateAt = :lastUpdateAt WHERE id = :id") - void updateHeartbeat(@Bind("id") String id, @Bind("lastUpdateAt") long lastUpdateAt); - - @SqlQuery("SELECT * FROM rdf_index_partition WHERE id = :id") - @RegisterRowMapper(RdfIndexPartitionMapper.class) - RdfIndexPartitionRecord findById(@Bind("id") String id); - - @SqlQuery( - "SELECT * FROM rdf_index_partition WHERE jobId = :jobId ORDER BY priority DESC, entityType, partitionIndex") - @RegisterRowMapper(RdfIndexPartitionMapper.class) - List findByJobId(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT COUNT(*) FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PENDING'") - int countPendingPartitions(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT COUNT(*) FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PROCESSING'") - int countInFlightPartitions(@Bind("jobId") String jobId); - - @ConnectionAwareSqlUpdate( - value = - "UPDATE rdf_index_partition p " - + "JOIN (SELECT id FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PENDING' " - + "AND claimableAt <= :now " - + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED) t ON p.id = t.id " - + "SET p.status = 'PROCESSING', p.assignedServer = :serverId, p.claimedAt = :now, " - + "p.startedAt = :now, p.lastUpdateAt = :now", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE rdf_index_partition SET status = 'PROCESSING', " - + "assignedServer = :serverId, claimedAt = :now, startedAt = :now, lastUpdateAt = :now " - + "WHERE id = (SELECT id FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PENDING' " - + "AND claimableAt <= :now " - + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED)", - connectionType = POSTGRES) - int claimNextPartitionAtomic( - @Bind("jobId") String jobId, @Bind("serverId") String serverId, @Bind("now") long now); - - @SqlQuery( - "SELECT * FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PROCESSING' " - + "AND assignedServer = :serverId AND claimedAt = :claimedAt " - + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1") - @RegisterRowMapper(RdfIndexPartitionMapper.class) - RdfIndexPartitionRecord findLatestClaimedPartition( - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("claimedAt") long claimedAt); - - @SqlUpdate( - "UPDATE rdf_index_partition SET status = 'PENDING', assignedServer = NULL, claimedAt = NULL, " - + "retryCount = retryCount + 1, lastError = 'Reclaimed due to stale heartbeat' " - + "WHERE jobId = :jobId AND status = 'PROCESSING' AND lastUpdateAt < :staleThreshold " - + "AND retryCount < :maxRetries") - int reclaimStalePartitionsForRetry( - @Bind("jobId") String jobId, - @Bind("staleThreshold") long staleThreshold, - @Bind("maxRetries") int maxRetries); - - @SqlUpdate( - "UPDATE rdf_index_partition SET status = 'FAILED', " - + "lastError = 'Exceeded max retries after stale heartbeat', completedAt = :now " - + "WHERE jobId = :jobId AND status = 'PROCESSING' AND lastUpdateAt < :staleThreshold " - + "AND retryCount >= :maxRetries") - int failStalePartitionsExceedingRetries( - @Bind("jobId") String jobId, - @Bind("staleThreshold") long staleThreshold, - @Bind("maxRetries") int maxRetries, - @Bind("now") long now); - - @SqlUpdate( - "UPDATE rdf_index_partition SET status = 'CANCELLED' WHERE jobId = :jobId AND status = 'PENDING'") - int cancelPendingPartitions(@Bind("jobId") String jobId); - - @SqlUpdate( - "UPDATE rdf_index_partition SET status = 'CANCELLED', " - + "lastError = 'Stopped by user', completedAt = :now, lastUpdateAt = :now " - + "WHERE jobId = :jobId AND status IN ('PENDING','PROCESSING')") - int cancelInFlightPartitions(@Bind("jobId") String jobId, @Bind("now") long now); - - @SqlQuery( - "SELECT COUNT(*) FROM rdf_index_partition " - + "WHERE jobId = :jobId AND status = 'PROCESSING' AND assignedServer = :serverId") - int countInFlightPartitionsForServer( - @Bind("jobId") String jobId, @Bind("serverId") String serverId); - - @SqlQuery("SELECT COUNT(*) FROM rdf_index_partition WHERE jobId = :jobId AND status = :status") - int countPartitionsByStatus(@Bind("jobId") String jobId, @Bind("status") String status); - - /** - * Status-guarded variant of {@link #update}: only writes if the row is still - * PROCESSING. Workers use this on completion so that a concurrent Stop - * (which moves the row to CANCELLED) isn't overwritten back to - * COMPLETED/FAILED, which would make the Stop button look unreliable. - * Returns the number of rows updated (0 means the row was no longer - * PROCESSING and the caller should skip side effects like server-stat - * increments). - */ - @SqlUpdate( - "UPDATE rdf_index_partition SET status = :status, processingCursor = :cursor, " - + "processedCount = :processedCount, successCount = :successCount, failedCount = :failedCount, " - + "assignedServer = :assignedServer, claimedAt = :claimedAt, startedAt = :startedAt, " - + "completedAt = :completedAt, lastUpdateAt = :lastUpdateAt, lastError = :lastError, " - + "retryCount = :retryCount WHERE id = :id AND status = 'PROCESSING'") - int updateIfProcessing( - @Bind("id") String id, - @Bind("status") String status, - @Bind("cursor") long cursor, - @Bind("processedCount") long processedCount, - @Bind("successCount") long successCount, - @Bind("failedCount") long failedCount, - @Bind("assignedServer") String assignedServer, - @Bind("claimedAt") Long claimedAt, - @Bind("startedAt") Long startedAt, - @Bind("completedAt") Long completedAt, - @Bind("lastUpdateAt") Long lastUpdateAt, - @Bind("lastError") String lastError, - @Bind("retryCount") int retryCount); - - @SqlUpdate( - "UPDATE rdf_index_partition SET status = :status, assignedServer = NULL, claimedAt = NULL, " - + "lastError = :reason, lastUpdateAt = :updatedAt, completedAt = :completedAt " - + "WHERE jobId = :jobId AND status = 'PROCESSING' AND assignedServer = :serverId") - int releaseProcessingPartitions( - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("status") String status, - @Bind("reason") String reason, - @Bind("updatedAt") long updatedAt, - @Bind("completedAt") Long completedAt); - - @SqlQuery( - "SELECT entityType, " - + "SUM(estimatedCount) as totalRecords, " - + "SUM(processedCount) as processedRecords, " - + "SUM(successCount) as successRecords, " - + "SUM(failedCount) as failedRecords, " - + "COUNT(*) as totalPartitions, " - + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " - + "SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) as failedPartitions " - + "FROM rdf_index_partition WHERE jobId = :jobId GROUP BY entityType") - @RegisterRowMapper(RdfEntityStatsMapper.class) - List getEntityStats(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT " - + "SUM(estimatedCount) as totalRecords, " - + "SUM(processedCount) as processedRecords, " - + "SUM(successCount) as successRecords, " - + "SUM(failedCount) as failedRecords, " - + "COUNT(*) as totalPartitions, " - + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " - + "SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) as failedPartitions, " - + "SUM(CASE WHEN status = 'PENDING' THEN 1 ELSE 0 END) as pendingPartitions, " - + "SUM(CASE WHEN status = 'PROCESSING' THEN 1 ELSE 0 END) as processingPartitions " - + "FROM rdf_index_partition WHERE jobId = :jobId") - @RegisterRowMapper(RdfAggregatedStatsMapper.class) - RdfAggregatedStatsRecord getAggregatedStats(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT assignedServer, " - + "SUM(processedCount) as processedRecords, " - + "SUM(successCount) as successRecords, " - + "SUM(failedCount) as failedRecords, " - + "COUNT(*) as totalPartitions, " - + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " - + "SUM(CASE WHEN status = 'PROCESSING' THEN 1 ELSE 0 END) as processingPartitions " - + "FROM rdf_index_partition WHERE jobId = :jobId AND assignedServer IS NOT NULL " - + "GROUP BY assignedServer") - @RegisterRowMapper(RdfServerStatsMapper.class) - List getServerStats(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT DISTINCT assignedServer FROM rdf_index_partition " - + "WHERE jobId = :jobId AND assignedServer IS NOT NULL") - List getAssignedServers(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT lastError FROM rdf_index_partition " - + "WHERE jobId = :jobId AND lastError IS NOT NULL " - + "ORDER BY lastUpdateAt DESC LIMIT :limit") - List findRecentPartitionErrors(@Bind("jobId") String jobId, @Bind("limit") int limit); - - @SqlUpdate("DELETE FROM rdf_index_partition") - void deleteAll(); - - class RdfIndexPartitionMapper implements RowMapper { - @Override - public RdfIndexPartitionRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new RdfIndexPartitionRecord( - rs.getString("id"), - rs.getString("jobId"), - rs.getString("entityType"), - rs.getInt("partitionIndex"), - rs.getLong("rangeStart"), - rs.getLong("rangeEnd"), - rs.getLong("estimatedCount"), - rs.getLong("workUnits"), - rs.getInt("priority"), - rs.getString("status"), - rs.getLong("processingCursor"), - rs.getLong("processedCount"), - rs.getLong("successCount"), - rs.getLong("failedCount"), - rs.getString("assignedServer"), - (Long) rs.getObject("claimedAt"), - (Long) rs.getObject("startedAt"), - (Long) rs.getObject("completedAt"), - (Long) rs.getObject("lastUpdateAt"), - rs.getString("lastError"), - rs.getInt("retryCount"), - rs.getLong("claimableAt")); - } - } + IndexMappingVersionDAO indexMappingVersionDAO(); - class RdfEntityStatsMapper implements RowMapper { - @Override - public RdfEntityStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new RdfEntityStatsRecord( - rs.getString("entityType"), - rs.getLong("totalRecords"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getInt("totalPartitions"), - rs.getInt("completedPartitions"), - rs.getInt("failedPartitions")); - } - } + @CreateSqlObject + AssetDAO assetDAO(); - class RdfAggregatedStatsMapper implements RowMapper { - @Override - public RdfAggregatedStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new RdfAggregatedStatsRecord( - rs.getLong("totalRecords"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getInt("totalPartitions"), - rs.getInt("completedPartitions"), - rs.getInt("failedPartitions"), - rs.getInt("pendingPartitions"), - rs.getInt("processingPartitions")); - } - } + @CreateSqlObject + DeletionLockDAO deletionLockDAO(); - class RdfServerStatsMapper implements RowMapper { - @Override - public RdfServerPartitionStatsRecord map(ResultSet rs, StatementContext ctx) - throws SQLException { - return new RdfServerPartitionStatsRecord( - rs.getString("assignedServer"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getInt("totalPartitions"), - rs.getInt("completedPartitions"), - rs.getInt("processingPartitions")); - } + class EntitiesCountRowMapper implements RowMapper { + @Override + public EntitiesCount map(ResultSet rs, StatementContext ctx) throws SQLException { + return new EntitiesCount() + .withTableCount(rs.getInt("tableCount")) + .withTopicCount(rs.getInt("topicCount")) + .withDashboardCount(rs.getInt("dashboardCount")) + .withPipelineCount(rs.getInt("pipelineCount")) + .withMlmodelCount(rs.getInt("mlmodelCount")) + .withServicesCount(rs.getInt("servicesCount")) + .withUserCount(rs.getInt("userCount")) + .withTeamCount(rs.getInt("teamCount")) + .withTestSuiteCount(rs.getInt("testSuiteCount")) + .withStorageContainerCount(rs.getInt("storageContainerCount")) + .withGlossaryCount(rs.getInt("glossaryCount")) + .withGlossaryTermCount(rs.getInt("glossaryTermCount")); } - - record RdfIndexPartitionRecord( - String id, - String jobId, - String entityType, - int partitionIndex, - long rangeStart, - long rangeEnd, - long estimatedCount, - long workUnits, - int priority, - String status, - long cursor, - long processedCount, - long successCount, - long failedCount, - String assignedServer, - Long claimedAt, - Long startedAt, - Long completedAt, - Long lastUpdateAt, - String lastError, - int retryCount, - long claimableAt) {} - - record RdfEntityStatsRecord( - String entityType, - long totalRecords, - long processedRecords, - long successRecords, - long failedRecords, - int totalPartitions, - int completedPartitions, - int failedPartitions) {} - - record RdfAggregatedStatsRecord( - long totalRecords, - long processedRecords, - long successRecords, - long failedRecords, - int totalPartitions, - int completedPartitions, - int failedPartitions, - int pendingPartitions, - int processingPartitions) {} - - record RdfServerPartitionStatsRecord( - String serverId, - long processedRecords, - long successRecords, - long failedRecords, - int totalPartitions, - int completedPartitions, - int processingPartitions) {} } - /** DAO for RDF distributed reindex lock. */ - interface RdfReindexLockDAO { - - @ConnectionAwareSqlUpdate( - value = - "INSERT IGNORE INTO rdf_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " - + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO rdf_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " - + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt) " - + "ON CONFLICT (lockKey) DO NOTHING", - connectionType = POSTGRES) - int insertIfNotExists( - @Bind("lockKey") String lockKey, - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("acquiredAt") long acquiredAt, - @Bind("lastHeartbeat") long lastHeartbeat, - @Bind("expiresAt") long expiresAt); - - @SqlUpdate( - "UPDATE rdf_reindex_lock SET lastHeartbeat = :lastHeartbeat, expiresAt = :expiresAt " - + "WHERE lockKey = :lockKey AND jobId = :jobId") - int updateHeartbeat( - @Bind("lockKey") String lockKey, - @Bind("jobId") String jobId, - @Bind("lastHeartbeat") long lastHeartbeat, - @Bind("expiresAt") long expiresAt); - - @SqlQuery("SELECT * FROM rdf_reindex_lock WHERE lockKey = :lockKey") - @RegisterRowMapper(RdfReindexLockMapper.class) - RdfReindexLockRecord findByKey(@Bind("lockKey") String lockKey); - - @SqlUpdate("DELETE FROM rdf_reindex_lock WHERE lockKey = :lockKey") - void delete(@Bind("lockKey") String lockKey); - - @SqlUpdate("DELETE FROM rdf_reindex_lock WHERE lockKey = :lockKey AND jobId = :jobId") - int deleteByKeyAndJob(@Bind("lockKey") String lockKey, @Bind("jobId") String jobId); - - @SqlUpdate("DELETE FROM rdf_reindex_lock WHERE expiresAt < :now") - int deleteExpiredLocks(@Bind("now") long now); - - @SqlUpdate( - "UPDATE rdf_reindex_lock SET jobId = :toJobId, serverId = :serverId, " - + "lastHeartbeat = :heartbeat, expiresAt = :expiresAt " - + "WHERE lockKey = :lockKey AND jobId = :fromJobId") - int updateLockOwner( - @Bind("lockKey") String lockKey, - @Bind("fromJobId") String fromJobId, - @Bind("toJobId") String toJobId, - @Bind("serverId") String serverId, - @Bind("heartbeat") long heartbeat, - @Bind("expiresAt") long expiresAt); - - default boolean tryAcquireLock( - String lockKey, String jobId, String serverId, long acquiredAt, long expiresAt) { - deleteExpiredLocks(System.currentTimeMillis()); - int inserted = insertIfNotExists(lockKey, jobId, serverId, acquiredAt, acquiredAt, expiresAt); - if (inserted > 0) { - return true; - } - - RdfReindexLockRecord existing = findByKey(lockKey); - if (existing != null && existing.isExpired()) { - delete(lockKey); - inserted = insertIfNotExists(lockKey, jobId, serverId, acquiredAt, acquiredAt, expiresAt); - return inserted > 0; - } - return false; - } - - default void releaseLock(String lockKey, String jobId) { - deleteByKeyAndJob(lockKey, jobId); - } - - default boolean transferLock( - String lockKey, - String fromJobId, - String toJobId, - String serverId, - long heartbeat, - long expiresAt) { - return updateLockOwner(lockKey, fromJobId, toJobId, serverId, heartbeat, expiresAt) > 0; - } - - class RdfReindexLockMapper implements RowMapper { - @Override - public RdfReindexLockRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new RdfReindexLockRecord( - rs.getString("lockKey"), - rs.getString("jobId"), - rs.getString("serverId"), - rs.getLong("acquiredAt"), - rs.getLong("lastHeartbeat"), - rs.getLong("expiresAt")); - } - } - - record RdfReindexLockRecord( - String lockKey, - String jobId, - String serverId, - long acquiredAt, - long lastHeartbeat, - long expiresAt) { - - public boolean isExpired() { - return System.currentTimeMillis() > expiresAt; - } + class ServicesCountRowMapper implements RowMapper { + @Override + public ServicesCount map(ResultSet rs, StatementContext ctx) throws SQLException { + return new ServicesCount() + .withDatabaseServiceCount(rs.getInt("databaseServiceCount")) + .withMessagingServiceCount(rs.getInt("messagingServiceCount")) + .withDashboardServiceCount(rs.getInt("dashboardServiceCount")) + .withPipelineServiceCount(rs.getInt("pipelineServiceCount")) + .withMlModelServiceCount(rs.getInt("mlModelServiceCount")) + .withStorageServiceCount(rs.getInt("storageServiceCount")); } } - /** DAO for RDF per-server distributed stats. */ - interface RdfIndexServerStatsDAO { - - record ServerStatsRecord( - String id, - String jobId, - String serverId, - String entityType, - long processedRecords, - long successRecords, - long failedRecords, - int partitionsCompleted, - int partitionsFailed, - long lastUpdatedAt) {} - - record AggregatedServerStats( - long processedRecords, - long successRecords, - long failedRecords, - int partitionsCompleted, - int partitionsFailed) {} - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO rdf_index_server_stats (id, jobId, serverId, entityType, processedRecords, " - + "successRecords, failedRecords, partitionsCompleted, partitionsFailed, lastUpdatedAt) " - + "VALUES (:id, :jobId, :serverId, :entityType, :processedRecords, :successRecords, " - + ":failedRecords, :partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " - + "ON DUPLICATE KEY UPDATE " - + "processedRecords = processedRecords + VALUES(processedRecords), " - + "successRecords = successRecords + VALUES(successRecords), " - + "failedRecords = failedRecords + VALUES(failedRecords), " - + "partitionsCompleted = partitionsCompleted + VALUES(partitionsCompleted), " - + "partitionsFailed = partitionsFailed + VALUES(partitionsFailed), " - + "lastUpdatedAt = VALUES(lastUpdatedAt)", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO rdf_index_server_stats (id, jobId, serverId, entityType, processedRecords, " - + "successRecords, failedRecords, partitionsCompleted, partitionsFailed, lastUpdatedAt) " - + "VALUES (:id, :jobId, :serverId, :entityType, :processedRecords, :successRecords, " - + ":failedRecords, :partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " - + "ON CONFLICT (jobId, serverId, entityType) DO UPDATE SET " - + "processedRecords = rdf_index_server_stats.processedRecords + EXCLUDED.processedRecords, " - + "successRecords = rdf_index_server_stats.successRecords + EXCLUDED.successRecords, " - + "failedRecords = rdf_index_server_stats.failedRecords + EXCLUDED.failedRecords, " - + "partitionsCompleted = rdf_index_server_stats.partitionsCompleted + EXCLUDED.partitionsCompleted, " - + "partitionsFailed = rdf_index_server_stats.partitionsFailed + EXCLUDED.partitionsFailed, " - + "lastUpdatedAt = EXCLUDED.lastUpdatedAt", - connectionType = POSTGRES) - void incrementStats( - @Bind("id") String id, - @Bind("jobId") String jobId, - @Bind("serverId") String serverId, - @Bind("entityType") String entityType, - @Bind("processedRecords") long processedRecords, - @Bind("successRecords") long successRecords, - @Bind("failedRecords") long failedRecords, - @Bind("partitionsCompleted") int partitionsCompleted, - @Bind("partitionsFailed") int partitionsFailed, - @Bind("lastUpdatedAt") long lastUpdatedAt); - - @SqlQuery("SELECT * FROM rdf_index_server_stats WHERE jobId = :jobId") - @RegisterRowMapper(RdfServerStatsRecordMapper.class) - List findByJobId(@Bind("jobId") String jobId); - - @SqlQuery( - "SELECT " - + "COALESCE(SUM(processedRecords), 0) as processedRecords, " - + "COALESCE(SUM(successRecords), 0) as successRecords, " - + "COALESCE(SUM(failedRecords), 0) as failedRecords, " - + "COALESCE(SUM(partitionsCompleted), 0) as partitionsCompleted, " - + "COALESCE(SUM(partitionsFailed), 0) as partitionsFailed " - + "FROM rdf_index_server_stats WHERE jobId = :jobId") - @RegisterRowMapper(RdfAggregatedServerStatsMapper.class) - AggregatedServerStats getAggregatedStats(@Bind("jobId") String jobId); - - @SqlUpdate("DELETE FROM rdf_index_server_stats") - void deleteAll(); + class ExecutionTrendRow { + private String dateKey; + private String status; + private Integer count; - class RdfServerStatsRecordMapper implements RowMapper { - @Override - public ServerStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { - return new ServerStatsRecord( - rs.getString("id"), - rs.getString("jobId"), - rs.getString("serverId"), - rs.getString("entityType"), - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getInt("partitionsCompleted"), - rs.getInt("partitionsFailed"), - rs.getLong("lastUpdatedAt")); - } - } + public ExecutionTrendRow() {} - class RdfAggregatedServerStatsMapper implements RowMapper { - @Override - public AggregatedServerStats map(ResultSet rs, StatementContext ctx) throws SQLException { - return new AggregatedServerStats( - rs.getLong("processedRecords"), - rs.getLong("successRecords"), - rs.getLong("failedRecords"), - rs.getInt("partitionsCompleted"), - rs.getInt("partitionsFailed")); - } + public ExecutionTrendRow(String dateKey, String status, Integer count) { + this.dateKey = dateKey; + this.status = status; + this.count = count; } - } - - @RegisterRowMapper(AuditLogRecordMapper.class) - interface AuditLogDAO { - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO audit_log_event(change_event_id, event_ts, event_type, user_name, " - + "actor_type, impersonated_by, service_name, " - + "entity_type, entity_id, entity_fqn, entity_fqn_hash, event_json, search_text, created_at) " - + "VALUES (:changeEventId::uuid, :eventTs, :eventType, :userName, " - + ":actorType, :impersonatedBy, :serviceName, " - + ":entityType, :entityId::uuid, :entityFQN, :entityFQNHash, :eventJson, :searchText, :createdAt) " - + "ON CONFLICT (change_event_id) DO NOTHING", - connectionType = POSTGRES) - @ConnectionAwareSqlUpdate( - value = - "INSERT IGNORE INTO audit_log_event(change_event_id, event_ts, event_type, user_name, " - + "actor_type, impersonated_by, service_name, " - + "entity_type, entity_id, entity_fqn, entity_fqn_hash, event_json, search_text, created_at) " - + "VALUES (:changeEventId, :eventTs, :eventType, :userName, " - + ":actorType, :impersonatedBy, :serviceName, " - + ":entityType, :entityId, :entityFQN, :entityFQNHash, :eventJson, :searchText, :createdAt)", - connectionType = MYSQL) - void insert(@BindBean AuditLogRecord record); - - @SqlQuery( - "SELECT id, change_event_id, event_ts, event_type, user_name, " - + "actor_type, impersonated_by, service_name, " - + "entity_type, entity_id, entity_fqn, entity_fqn_hash, event_json, search_text, created_at " - + "FROM audit_log_event LIMIT :limit") - List list( - @Define("condition") String condition, - @Define("orderClause") String orderClause, - @Bind("userName") String userName, - @Bind("actorType") String actorType, - @Bind("serviceName") String serviceName, - @Bind("entityType") String entityType, - @Bind("entityFQN") String entityFQN, - @Bind("entityFQNHASH") String entityFqnHash, - @Bind("eventType") String eventType, - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs, - @Bind("searchTerm") String searchTerm, - @Bind("afterEventTs") Long afterEventTs, - @Bind("afterId") Long afterId, - @Bind("limit") int limit); - - @SqlQuery("SELECT COUNT(id) FROM audit_log_event ") - int count( - @Define("condition") String condition, - @Bind("userName") String userName, - @Bind("actorType") String actorType, - @Bind("serviceName") String serviceName, - @Bind("entityType") String entityType, - @Bind("entityFQN") String entityFQN, - @Bind("entityFQNHASH") String entityFqnHash, - @Bind("eventType") String eventType, - @Bind("startTs") Long startTs, - @Bind("endTs") Long endTs, - @Bind("searchTerm") String searchTerm); - - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM audit_log_event " - + "WHERE created_at < :cutoffTs ORDER BY created_at LIMIT :limit", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM audit_log_event " - + "WHERE ctid IN ( " - + " SELECT ctid FROM audit_log_event " - + " WHERE created_at < :cutoffTs ORDER BY created_at LIMIT :limit " - + ")", - connectionType = POSTGRES) - int deleteInBatches(@Bind("cutoffTs") long cutoffTs, @Bind("limit") int limit); - } - - // OAuth 2.0 DAOs for MCP Server - interface OAuthClientDAO { - @SqlQuery( - "SELECT id, client_id, client_secret_encrypted, client_name, redirect_uris, grant_types, token_endpoint_auth_method, scopes FROM oauth_clients WHERE client_id = :clientId") - @RegisterRowMapper(OAuthClientRowMapper.class) - OAuthRecords.OAuthClientRecord findByClientId(@Bind("clientId") String clientId); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO oauth_clients (client_id, client_secret_encrypted, client_name, redirect_uris, grant_types, token_endpoint_auth_method, scopes) VALUES (:clientId, :clientSecret, :clientName, :redirectUris ::jsonb, :grantTypes ::jsonb, :authMethod, :scopes ::jsonb)", - connectionType = POSTGRES) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO oauth_clients (client_id, client_secret_encrypted, client_name, redirect_uris, grant_types, token_endpoint_auth_method, scopes) VALUES (:clientId, :clientSecret, :clientName, :redirectUris, :grantTypes, :authMethod, :scopes)", - connectionType = MYSQL) - void insert( - @Bind("clientId") String clientId, - @Bind("clientSecret") String clientSecret, - @Bind("clientName") String clientName, - @Bind("redirectUris") String redirectUris, - @Bind("grantTypes") String grantTypes, - @Bind("authMethod") String authMethod, - @Bind("scopes") String scopes); - - @SqlUpdate("DELETE FROM oauth_clients WHERE client_id = :clientId") - void delete(@Bind("clientId") String clientId); - } - - interface OAuthAuthorizationCodeDAO { - @SqlQuery( - "SELECT code, client_id, user_name, code_challenge, code_challenge_method, redirect_uri, scopes, expires_at, used FROM oauth_authorization_codes WHERE code = :code") - @RegisterRowMapper(OAuthAuthorizationCodeRowMapper.class) - OAuthRecords.OAuthAuthorizationCodeRecord findByCode(@Bind("code") String code); - - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO oauth_authorization_codes (code, client_id, user_name, code_challenge, code_challenge_method, redirect_uri, scopes, expires_at) VALUES (:code, :clientId, :userName, :codeChallenge, :codeChallengeMethod, :redirectUri, :scopes ::jsonb, :expiresAt)", - connectionType = POSTGRES) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO oauth_authorization_codes (code, client_id, user_name, code_challenge, code_challenge_method, redirect_uri, scopes, expires_at) VALUES (:code, :clientId, :userName, :codeChallenge, :codeChallengeMethod, :redirectUri, :scopes, :expiresAt)", - connectionType = MYSQL) - void insert( - @Bind("code") String code, - @Bind("clientId") String clientId, - @Bind("userName") String userName, - @Bind("codeChallenge") String codeChallenge, - @Bind("codeChallengeMethod") String codeChallengeMethod, - @Bind("redirectUri") String redirectUri, - @Bind("scopes") String scopes, - @Bind("expiresAt") long expiresAt); - @SqlUpdate( - "UPDATE oauth_authorization_codes SET used = TRUE WHERE code = :code AND used = FALSE") - int markAsUsedAtomic(@Bind("code") String code); - - @SqlUpdate("DELETE FROM oauth_authorization_codes WHERE code = :code") - void delete(@Bind("code") String code); + public String getDateKey() { + return dateKey; + } - @SqlUpdate("DELETE FROM oauth_authorization_codes WHERE expires_at < :currentTime") - void deleteExpired(@Bind("currentTime") long currentTime); - } + public void setDateKey(String dateKey) { + this.dateKey = dateKey; + } - interface OAuthAccessTokenDAO { - @SqlQuery( - "SELECT id, token_hash, access_token_encrypted, client_id, user_name, scopes, expires_at FROM oauth_access_tokens WHERE token_hash = :tokenHash") - @RegisterRowMapper(OAuthAccessTokenRowMapper.class) - OAuthRecords.OAuthAccessTokenRecord findByTokenHash(@Bind("tokenHash") String tokenHash); + public String getStatus() { + return status; + } - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO oauth_access_tokens (token_hash, access_token_encrypted, client_id, user_name, scopes, expires_at) VALUES (:tokenHash, :accessTokenEncrypted, :clientId, :userName, :scopes ::jsonb, :expiresAt)", - connectionType = POSTGRES) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO oauth_access_tokens (token_hash, access_token_encrypted, client_id, user_name, scopes, expires_at) VALUES (:tokenHash, :accessTokenEncrypted, :clientId, :userName, :scopes, :expiresAt)", - connectionType = MYSQL) - void insert( - @Bind("tokenHash") String tokenHash, - @Bind("accessTokenEncrypted") String accessTokenEncrypted, - @Bind("clientId") String clientId, - @Bind("userName") String userName, - @Bind("scopes") String scopes, - @Bind("expiresAt") long expiresAt); + public void setStatus(String status) { + this.status = status; + } - @SqlUpdate("DELETE FROM oauth_access_tokens WHERE token_hash = :tokenHash") - void delete(@Bind("tokenHash") String tokenHash); + public Integer getCount() { + return count; + } - @SqlUpdate("DELETE FROM oauth_access_tokens WHERE expires_at < :currentTime") - void deleteExpired(@Bind("currentTime") long currentTime); + public void setCount(Integer count) { + this.count = count; + } } - interface OAuthRefreshTokenDAO { - @SqlQuery( - "SELECT id, token_hash, refresh_token_encrypted, client_id, user_name, scopes, expires_at, revoked FROM oauth_refresh_tokens WHERE token_hash = :tokenHash") - @RegisterRowMapper(OAuthRefreshTokenRowMapper.class) - OAuthRecords.OAuthRefreshTokenRecord findByTokenHash(@Bind("tokenHash") String tokenHash); + class ExecutionTrendRowMapper implements RowMapper { + @Override + public ExecutionTrendRow map(ResultSet rs, StatementContext ctx) throws SQLException { + ExecutionTrendRow row = new ExecutionTrendRow(); + row.setDateKey(rs.getString("date_key")); + row.setStatus(rs.getString("status")); + row.setCount(rs.getInt("count")); + return row; + } + } - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO oauth_refresh_tokens (token_hash, refresh_token_encrypted, client_id, user_name, scopes, expires_at) VALUES (:tokenHash, :refreshTokenEncrypted, :clientId, :userName, :scopes ::jsonb, :expiresAt)", - connectionType = POSTGRES) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO oauth_refresh_tokens (token_hash, refresh_token_encrypted, client_id, user_name, scopes, expires_at) VALUES (:tokenHash, :refreshTokenEncrypted, :clientId, :userName, :scopes, :expiresAt)", - connectionType = MYSQL) - void insert( - @Bind("tokenHash") String tokenHash, - @Bind("refreshTokenEncrypted") String refreshTokenEncrypted, - @Bind("clientId") String clientId, - @Bind("userName") String userName, - @Bind("scopes") String scopes, - @Bind("expiresAt") long expiresAt); + class RuntimeTrendRow { + private String dateKey; + private Long firstTimestamp; + private Double maxRuntime; + private Double minRuntime; + private Double avgRuntime; + private Integer totalPipelines; - @SqlUpdate("UPDATE oauth_refresh_tokens SET revoked = TRUE WHERE token_hash = :tokenHash") - void revoke(@Bind("tokenHash") String tokenHash); + public RuntimeTrendRow() {} - @SqlUpdate( - "UPDATE oauth_refresh_tokens SET revoked = TRUE WHERE token_hash = :tokenHash AND revoked = FALSE") - int revokeAtomic(@Bind("tokenHash") String tokenHash); + public RuntimeTrendRow( + String dateKey, + Long firstTimestamp, + Double maxRuntime, + Double minRuntime, + Double avgRuntime, + Integer totalPipelines) { + this.dateKey = dateKey; + this.firstTimestamp = firstTimestamp; + this.maxRuntime = maxRuntime; + this.minRuntime = minRuntime; + this.avgRuntime = avgRuntime; + this.totalPipelines = totalPipelines; + } - @SqlUpdate("DELETE FROM oauth_refresh_tokens WHERE token_hash = :tokenHash") - void delete(@Bind("tokenHash") String tokenHash); + public String getDateKey() { + return dateKey; + } - @SqlUpdate("DELETE FROM oauth_refresh_tokens WHERE expires_at < :currentTime") - void deleteExpired(@Bind("currentTime") long currentTime); + public void setDateKey(String dateKey) { + this.dateKey = dateKey; + } - @SqlUpdate( - "UPDATE oauth_refresh_tokens SET revoked = TRUE WHERE client_id = :clientId AND user_name = :userName AND revoked = FALSE") - void revokeAllForUser(@Bind("clientId") String clientId, @Bind("userName") String userName); - } + public Long getFirstTimestamp() { + return firstTimestamp; + } - interface McpPendingAuthRequestDAO { - @SqlQuery( - "SELECT auth_request_id, client_id, code_challenge, code_challenge_method, redirect_uri, mcp_state, scopes, pac4j_state, pac4j_nonce, pac4j_code_verifier, expires_at FROM mcp_pending_auth_requests WHERE auth_request_id = :authRequestId") - @RegisterRowMapper(McpPendingAuthRequestRowMapper.class) - OAuthRecords.McpPendingAuthRequest findByAuthRequestId( - @Bind("authRequestId") String authRequestId); + public void setFirstTimestamp(Long firstTimestamp) { + this.firstTimestamp = firstTimestamp; + } - @SqlQuery( - "SELECT auth_request_id, client_id, code_challenge, code_challenge_method, redirect_uri, mcp_state, scopes, pac4j_state, pac4j_nonce, pac4j_code_verifier, expires_at FROM mcp_pending_auth_requests WHERE pac4j_state = :pac4jState") - @RegisterRowMapper(McpPendingAuthRequestRowMapper.class) - OAuthRecords.McpPendingAuthRequest findByPac4jState(@Bind("pac4jState") String pac4jState); + public Double getMaxRuntime() { + return maxRuntime; + } - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO mcp_pending_auth_requests (auth_request_id, client_id, code_challenge, code_challenge_method, redirect_uri, mcp_state, scopes, pac4j_state, pac4j_nonce, pac4j_code_verifier, expires_at) VALUES (:authRequestId, :clientId, :codeChallenge, :codeChallengeMethod, :redirectUri, :mcpState, :scopes ::jsonb, :pac4jState, :pac4jNonce, :pac4jCodeVerifier, :expiresAt)", - connectionType = POSTGRES) - @ConnectionAwareSqlUpdate( - value = - "INSERT INTO mcp_pending_auth_requests (auth_request_id, client_id, code_challenge, code_challenge_method, redirect_uri, mcp_state, scopes, pac4j_state, pac4j_nonce, pac4j_code_verifier, expires_at) VALUES (:authRequestId, :clientId, :codeChallenge, :codeChallengeMethod, :redirectUri, :mcpState, :scopes, :pac4jState, :pac4jNonce, :pac4jCodeVerifier, :expiresAt)", - connectionType = MYSQL) - void insert( - @Bind("authRequestId") String authRequestId, - @Bind("clientId") String clientId, - @Bind("codeChallenge") String codeChallenge, - @Bind("codeChallengeMethod") String codeChallengeMethod, - @Bind("redirectUri") String redirectUri, - @Bind("mcpState") String mcpState, - @Bind("scopes") String scopes, - @Bind("pac4jState") String pac4jState, - @Bind("pac4jNonce") String pac4jNonce, - @Bind("pac4jCodeVerifier") String pac4jCodeVerifier, - @Bind("expiresAt") long expiresAt); + public void setMaxRuntime(Double maxRuntime) { + this.maxRuntime = maxRuntime; + } - @SqlUpdate( - "UPDATE mcp_pending_auth_requests SET pac4j_state = :pac4jState, pac4j_nonce = :pac4jNonce, pac4j_code_verifier = :pac4jCodeVerifier WHERE auth_request_id = :authRequestId") - void updatePac4jSession( - @Bind("authRequestId") String authRequestId, - @Bind("pac4jState") String pac4jState, - @Bind("pac4jNonce") String pac4jNonce, - @Bind("pac4jCodeVerifier") String pac4jCodeVerifier); + public Double getMinRuntime() { + return minRuntime; + } - @SqlUpdate("DELETE FROM mcp_pending_auth_requests WHERE auth_request_id = :authRequestId") - void delete(@Bind("authRequestId") String authRequestId); + public void setMinRuntime(Double minRuntime) { + this.minRuntime = minRuntime; + } - @SqlUpdate("DELETE FROM mcp_pending_auth_requests WHERE expires_at < :currentTime") - void deleteExpired(@Bind("currentTime") long currentTime); - } + public Double getAvgRuntime() { + return avgRuntime; + } - class McpPendingAuthRequestRowMapper implements RowMapper { - @Override - public OAuthRecords.McpPendingAuthRequest map(ResultSet rs, StatementContext ctx) - throws SQLException { - String scopesJson = rs.getString("scopes"); - List scopes = - scopesJson != null - ? org.openmetadata.schema.utils.JsonUtils.readValue( - scopesJson, new com.fasterxml.jackson.core.type.TypeReference>() {}) - : List.of(); - return new OAuthRecords.McpPendingAuthRequest( - rs.getString("auth_request_id"), - rs.getString("client_id"), - rs.getString("code_challenge"), - rs.getString("code_challenge_method"), - rs.getString("redirect_uri"), - rs.getString("mcp_state"), - scopes, - rs.getString("pac4j_state"), - rs.getString("pac4j_nonce"), - rs.getString("pac4j_code_verifier"), - rs.getLong("expires_at")); + public void setAvgRuntime(Double avgRuntime) { + this.avgRuntime = avgRuntime; } - } - interface FolderDAO extends EntityDAO { - @Override - default String getTableName() { - return "drive_folder"; + public Integer getTotalPipelines() { + return totalPipelines; } - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.data.Folder.class; + public void setTotalPipelines(Integer totalPipelines) { + this.totalPipelines = totalPipelines; } + } + class RuntimeTrendRowMapper implements RowMapper { @Override - default String getNameHashColumn() { - return "nameHash"; + public RuntimeTrendRow map(ResultSet rs, StatementContext ctx) throws SQLException { + RuntimeTrendRow row = new RuntimeTrendRow(); + row.setDateKey(rs.getString("date_key")); + row.setFirstTimestamp(rs.getLong("first_timestamp")); + row.setMaxRuntime(rs.getDouble("max_runtime")); + row.setMinRuntime(rs.getDouble("min_runtime")); + row.setAvgRuntime(rs.getDouble("avg_runtime")); + row.setTotalPipelines(rs.getInt("total_pipelines")); + return row; } } - interface ContextFileDAO extends EntityDAO { - @Override - default String getTableName() { - return "context_file"; + class ServiceBreakdownRow { + private String serviceType; + private Integer pipelineCount; + + public ServiceBreakdownRow() {} + + public ServiceBreakdownRow(String serviceType, Integer pipelineCount) { + this.serviceType = serviceType; + this.pipelineCount = pipelineCount; } - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.data.ContextFile.class; + public String getServiceType() { + return serviceType; } - @Override - default String getNameHashColumn() { - return "nameHash"; + public void setServiceType(String serviceType) { + this.serviceType = serviceType; } - } - interface ContextFileContentDAO - extends EntityDAO { - @Override - default String getTableName() { - return "context_file_content"; + public Integer getPipelineCount() { + return pipelineCount; } - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.data.ContextFileContent.class; + public void setPipelineCount(Integer pipelineCount) { + this.pipelineCount = pipelineCount; } + } + class ServiceBreakdownRowMapper implements RowMapper { @Override - default String getNameHashColumn() { - return "nameHash"; + public ServiceBreakdownRow map(ResultSet rs, StatementContext ctx) throws SQLException { + ServiceBreakdownRow row = new ServiceBreakdownRow(); + row.setServiceType(rs.getString("service_type")); + row.setPipelineCount(rs.getInt("pipeline_count")); + return row; } - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM context_file_content " - + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.contextFile.id')) = :contextFileId", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM context_file_content " - + "WHERE json->'contextFile'->>'id' = :contextFileId", - connectionType = POSTGRES) - List listByContextFileId(@Bind("contextFileId") String contextFileId); } - interface KnowledgePageDAO extends EntityDAO { - String KNOWLEDGE_PAGE_ENTITY = "page"; + class PipelineMetricsRow { + private Integer totalPipelines; + private Integer activePipelines; + private Integer successfulPipelines; + private Integer failedPipelines; + + public PipelineMetricsRow() {} - @Override - default String getTableName() { - return "knowledge_center"; + public PipelineMetricsRow( + Integer totalPipelines, + Integer activePipelines, + Integer successfulPipelines, + Integer failedPipelines) { + this.totalPipelines = totalPipelines; + this.activePipelines = activePipelines; + this.successfulPipelines = successfulPipelines; + this.failedPipelines = failedPipelines; } - @Override - default Class getEntityClass() { - return org.openmetadata.schema.entity.data.Page.class; + public Integer getTotalPipelines() { + return totalPipelines; } - @Override - default String getNameHashColumn() { - return "fqnHash"; + public void setTotalPipelines(Integer totalPipelines) { + this.totalPipelines = totalPipelines; } - @Override - default boolean supportsSoftDelete() { - return false; + public Integer getActivePipelines() { + return activePipelines; } - /** - * When the caller supplies {@code entityId} + {@code entityType} (e.g. from a data-asset - * page that wants the list of knowledge pages referencing it), join against - * {@code entity_relationship} so that only pages whose {@code relatedEntities} contains - * the target entity are returned. Without this override, the base {@code EntityDAO.listAfter} - * ignores those params and returns every knowledge page — breaking the Knowledge - * Articles right-panel widget (and the corresponding playwright assertions). - */ - @Override - default int listCount(ListFilter filter) { - String entityId = filter.getQueryParam("entityId"); - String entityType = filter.getQueryParam("entityType"); - String knowledgePageType = filter.getQueryParam("pageType"); - String tagFQN = filter.getQueryParam("tagFQN"); - String tagListCondition = - "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; - String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; - if (nullOrEmpty(tagFQN)) { - tagListCondition = ""; - tagFilterCondition = "WHERE"; - } - Map bindMap = new HashMap<>(); - if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { - String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); - String condition = - String.format( - "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s " - + "entity_relationship.fromId IN (%s) %s" - + "and entity_relationship.toEntity = :toEntityType %s", - tagListCondition, - tagFilterCondition, - entityId, - getRelationCondition(entityType), - knowledgePageTypeQuery); - bindMap.put("toEntityType", KNOWLEDGE_PAGE_ENTITY); - bindMap.put("tagFQN", tagFQN); - if (!nullOrEmpty(knowledgePageTypeQuery)) { - bindMap.put("pageType", knowledgePageType); - } - return listKnowledgePageCountByEntity(condition, bindMap); - } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) - || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { - throw new IllegalArgumentException( - "Query Param Entity Id and Entity Type both needs to be provided."); - } + public void setActivePipelines(Integer activePipelines) { + this.activePipelines = activePipelines; + } - String knowledgePageQueryClause = - String.format( - "%s %s %s", - tagListCondition, - tagFilterCondition, - getKnowledgePageTypeQuery("", knowledgePageType)); - return listCount( - getTableName(), - getNameHashColumn(), - filter.getQueryParams(), - getKnowledgePageWhereClause(knowledgePageQueryClause)); + public Integer getSuccessfulPipelines() { + return successfulPipelines; } - @Override - default List listBefore( - ListFilter filter, int limit, String beforeName, String beforeId) { - String entityId = filter.getQueryParam("entityId"); - String entityType = filter.getQueryParam("entityType"); - String knowledgePageType = filter.getQueryParam("pageType"); - String tagFQN = filter.getQueryParam("tagFQN"); - String tagListCondition = - "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; - String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; - if (nullOrEmpty(tagFQN)) { - tagListCondition = ""; - tagFilterCondition = "WHERE"; - } - Map bindMap = new HashMap<>(); - if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { - String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); - String condition = - String.format( - "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s entity_relationship.fromId IN (%s) " - + "%s and entity_relationship.toEntity = :toEntity %s " - + "and (knowledge_center.name < :beforeName OR (knowledge_center.name = :beforeName AND knowledge_center.id < :beforeId)) order by knowledge_center.name DESC,knowledge_center.id DESC LIMIT :limit", - tagListCondition, - tagFilterCondition, - entityId, - getRelationCondition(entityType), - knowledgePageTypeQuery); - bindMap.put("toEntity", KNOWLEDGE_PAGE_ENTITY); - bindMap.put("beforeName", beforeName); - bindMap.put("beforeId", beforeId); - bindMap.put("limit", limit); - bindMap.put("tagFQN", tagFQN); - if (!nullOrEmpty(knowledgePageTypeQuery)) { - bindMap.put("pageType", knowledgePageType); - } - return listBeforeKnowledgePageByEntityId(condition, bindMap); - } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) - || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { - throw new IllegalArgumentException( - "Query Param Entity Id and Entity Type both needs to be provided."); - } - String knowledgePageQueryClause = - String.format( - "%s %s %s", - tagListCondition, - tagFilterCondition, - getKnowledgePageTypeQuery("", knowledgePageType)); - beforeName = FullyQualifiedName.unquoteName(beforeName); - return listBefore( - getTableName(), - filter.getQueryParams(), - getKnowledgePageWhereClause(knowledgePageQueryClause), - limit, - beforeName, - beforeId); + public void setSuccessfulPipelines(Integer successfulPipelines) { + this.successfulPipelines = successfulPipelines; } - @Override - default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { - String entityId = filter.getQueryParam("entityId"); - String entityType = filter.getQueryParam("entityType"); - String knowledgePageType = filter.getQueryParam("pageType"); - String tagFQN = filter.getQueryParam("tagFQN"); - String tagListCondition = - "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; - String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; - if (nullOrEmpty(tagFQN)) { - tagListCondition = ""; - tagFilterCondition = "WHERE"; - } - Map bindMap = new HashMap<>(); - if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { - String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); - String condition = - String.format( - "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s entity_relationship.fromId IN (%s) " - + "%s and entity_relationship.toEntity = :toEntity %s " - + "and (knowledge_center.name > :afterName OR (knowledge_center.name = :afterName AND knowledge_center.id > :afterId)) order by knowledge_center.name ASC,knowledge_center.id ASC LIMIT :limit", - tagListCondition, - tagFilterCondition, - entityId, - getRelationCondition(entityType), - knowledgePageTypeQuery); - bindMap.put("toEntity", KNOWLEDGE_PAGE_ENTITY); - bindMap.put("afterName", afterName); - bindMap.put("afterId", afterId); - bindMap.put("limit", limit); - bindMap.put("tagFQN", tagFQN); - if (!nullOrEmpty(knowledgePageTypeQuery)) { - bindMap.put("pageType", knowledgePageType); - } - return listAfterKnowledgePageByEntityId(condition, bindMap); - } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) - || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { - throw new IllegalArgumentException( - "Query Param Entity Id and Entity Type both needs to be provided."); - } - String knowledgePageQueryClause = - String.format( - "%s %s %s", - tagListCondition, - tagFilterCondition, - getKnowledgePageTypeQuery("", knowledgePageType)); - afterName = FullyQualifiedName.unquoteName(afterName); - return listAfter( - getTableName(), - filter.getQueryParams(), - getKnowledgePageWhereClause(knowledgePageQueryClause), - limit, - afterName, - afterId); + public Integer getFailedPipelines() { + return failedPipelines; } - private String getRelationCondition(String entityType) { - // Users/teams "own" pages (membership-based); every other entity type reaches the page - // through a HAS relationship (the page's relatedEntities list). - String owns = String.valueOf(OWNS.ordinal()); - String has = String.valueOf(HAS.ordinal()); - if (entityType.equals(USER) || entityType.equals(TEAM)) { - return String.format(" and entity_relationship.relation = %s ", owns); - } else { - return String.format(" and entity_relationship.relation = %s ", has); - } + public void setFailedPipelines(Integer failedPipelines) { + this.failedPipelines = failedPipelines; } + } - private String getKnowledgePageWhereClause(String knowledgePageQueryClause) { - return nullOrEmpty(knowledgePageQueryClause) ? "WHERE TRUE" : knowledgePageQueryClause; + class PipelineMetricsRowMapper implements RowMapper { + @Override + public PipelineMetricsRow map(ResultSet rs, StatementContext ctx) throws SQLException { + PipelineMetricsRow row = new PipelineMetricsRow(); + row.setTotalPipelines(rs.getInt("total_pipelines")); + row.setActivePipelines(rs.getInt("active_pipelines")); + row.setSuccessfulPipelines(rs.getInt("successful_pipelines")); + row.setFailedPipelines(rs.getInt("failed_pipelines")); + return row; } + } - private String getKnowledgePageTypeQuery(String clause, String type) { - if (!nullOrEmpty(type)) { - if (Boolean.TRUE.equals( - org.openmetadata.service.resources.databases.DatasourceConfig.getInstance() - .isMySQL())) { - return String.format( - " %s JSON_EXTRACT(knowledge_center.json, '$.pageType') = :pageType", clause); - } else { - return String.format(" %s knowledge_center.json->>'pageType' = :pageType", clause); - } - } - if ("AND".equals(clause)) { - return ""; - } - return "TRUE"; + class PipelineSummaryRow { + private String id; + private String json; + private String latestStatus; + + public PipelineSummaryRow() {} + + public PipelineSummaryRow(String id, String json, String latestStatus) { + this.id = id; + this.json = json; + this.latestStatus = latestStatus; } - @SqlQuery("SELECT knowledge_center.json FROM knowledge_center ") - List listAfterKnowledgePageByEntityId( - @Define("cond") String cond, @BindMap Map bindings); + public String getId() { + return id; + } - @SqlQuery( - "SELECT json FROM (SELECT knowledge_center.name,knowledge_center.id, knowledge_center.json FROM knowledge_center ) last_rows_subquery ORDER BY name,id") - List listBeforeKnowledgePageByEntityId( - @Define("cond") String cond, @BindMap Map bindings); + public void setId(String id) { + this.id = id; + } - @SqlQuery("SELECT count(*) FROM knowledge_center ") - int listKnowledgePageCountByEntity( - @Define("cond") String cond, @BindMap Map bindings); + public String getJson() { + return json; + } - @SqlQuery( - "SELECT json " - + "FROM knowledge_center " - + "WHERE id NOT IN (" - + " SELECT toId FROM entity_relationship WHERE (relation = 0 AND toEntity = 'page') OR (relation = 9 AND toEntity = 'page')" - + ")") - List listTopLevelPages(); + public void setJson(String json) { + this.json = json; + } - @SqlQuery( - "SELECT kc.json " - + "FROM knowledge_center kc " - + "JOIN entity_relationship er ON kc.id = er.toId " - + "WHERE er.fromId = :parentId " - + "AND (er.relation = 9 or er.relation = 0) " - + "AND er.toEntity = 'page'") - List listChildren(@Bind("parentId") String parentId); + public String getLatestStatus() { + return latestStatus; + } - @ConnectionAwareSqlUpdate( - value = "UPDATE knowledge_center SET json = :json, fqnHash = :fqnHash WHERE id = :id", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "UPDATE knowledge_center SET json = :json::jsonb, fqnHash = :fqnHash WHERE id = :id", - connectionType = POSTGRES) - void updateFullyQualifiedName( - @Bind("id") String pageId, @Bind("json") String json, @BindFQN("fqnHash") String fqnHash); + public void setLatestStatus(String latestStatus) { + this.latestStatus = latestStatus; + } + } + + class PipelineSummaryRowMapper implements RowMapper { + @Override + public PipelineSummaryRow map(ResultSet rs, StatementContext ctx) throws SQLException { + PipelineSummaryRow row = new PipelineSummaryRow(); + row.setId(rs.getString("id")); + row.setJson(rs.getString("json")); + row.setLatestStatus(rs.getString("latest_status")); + return row; + } } interface AssetDAO { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java index 7ff7a0b54589..78c6f31aec1f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ContainerRepository.java @@ -262,7 +262,7 @@ private Map batchFetchServices(List containers // De-dupe service IDs before resolving them to references. In any practical paged // listing the children are all under the same storage service, so the naive loop // below would call getEntityReferenceById N times for the same service id — - // each call hits CACHE_WITH_ID (or DB) for the full StorageService JSON. Cache one + // each call hits EntityCaches.CACHE_WITH_ID (or DB) for the full StorageService JSON. Cache one // ref per unique service id and fan it back out to every child. Map serviceRefById = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : records) { @@ -1346,8 +1346,8 @@ private void updateParent(Container original, Container updated) { validateSubtreeSize(oldFqn, descendantCount, maxAllowed); List renamedContainers = - invalidateCacheForRenameCascade(CONTAINER, oldFqn); - invalidateCacheForTaggedEntitiesAndDescendants(CONTAINER, oldFqn); + EntityCacheInvalidator.invalidateCacheForRenameCascade(CONTAINER, oldFqn); + EntityCacheInvalidator.invalidateCacheForTaggedEntitiesAndDescendants(CONTAINER, oldFqn); daoCollection.containerDAO().updateFqn(oldFqn, newFqn); @@ -1377,7 +1377,7 @@ private void updateParent(Container original, Container updated) { FIELD_PARENT, original.getParent(), updated.getParent(), true, entityReferenceMatch); updateAssetIndexes(oldFqn, newFqn); - finishInvalidateCacheForRenameCascade(CONTAINER, renamedContainers); + EntityCacheInvalidator.finishInvalidateCacheForRenameCascade(CONTAINER, renamedContainers); } private void updateParentRelationship(Container orig, Container updated) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CoreRelationshipDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CoreRelationshipDAOs.java new file mode 100644 index 000000000000..d9097dee14f1 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CoreRelationshipDAOs.java @@ -0,0 +1,1583 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.ListFilter.escapeApostrophe; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.Triple; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindBean; +import org.jdbi.v3.sqlobject.customizer.BindBeanList; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.customizer.BindMap; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.jdbi.v3.sqlobject.statement.UseRowMapper; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.analytics.ReportData; +import org.openmetadata.schema.entity.data.Query; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlBatch; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.util.EntityUtil; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.jdbi.BindConcat; +import org.openmetadata.service.util.jdbi.BindFQN; +import org.openmetadata.service.util.jdbi.BindUUID; + +public interface CoreRelationshipDAOs { + @CreateSqlObject + EntityRelationshipDAO relationshipDAO(); + + @CreateSqlObject + FieldRelationshipDAO fieldRelationshipDAO(); + + @CreateSqlObject + EntityExtensionDAO entityExtensionDAO(); + + interface EntityExtensionDAO { + @ConnectionAwareSqlUpdate( + value = + "REPLACE INTO entity_extension(id, extension, jsonSchema, json) " + + "VALUES (:id, :extension, :jsonSchema, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_extension(id, extension, jsonSchema, json) " + + "VALUES (:id, :extension, :jsonSchema, (:json :: jsonb)) " + + "ON CONFLICT (id, extension) DO UPDATE SET jsonSchema = EXCLUDED.jsonSchema, json = EXCLUDED.json", + connectionType = POSTGRES) + void insert( + @BindUUID("id") UUID id, + @Bind("extension") String extension, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json); + + @Transaction + @ConnectionAwareSqlBatch( + value = + "REPLACE INTO entity_extension(id, extension, jsonSchema, json) " + + "VALUES (:id, :extension, :jsonSchema, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlBatch( + value = + "INSERT INTO entity_extension(id, extension, jsonSchema, json) " + + "VALUES (:id, :extension, :jsonSchema, (:json :: jsonb)) " + + "ON CONFLICT (id, extension) DO UPDATE SET jsonSchema = EXCLUDED.jsonSchema, json = EXCLUDED.json", + connectionType = POSTGRES) + void insertMany( + @BindUUID("id") List id, + @Bind("extension") List extension, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") List json); + + @SqlQuery("SELECT json FROM entity_extension WHERE id = :id AND extension = :extension") + String getExtension(@BindUUID("id") UUID id, @Bind("extension") String extension); + + @SqlQuery( + "SELECT id, extension, json " + + "FROM entity_extension " + + "WHERE id IN () AND extension LIKE :extension " + + "ORDER BY id, extension") + @RegisterRowMapper(ExtensionRecordWithIdMapper.class) + List getExtensionsBatch( + @BindList("ids") List ids, + @BindConcat( + value = "extension", + parts = {":extensionPrefix", ".%"}) + String extensionPrefix); + + @SqlQuery( + "SELECT id, extension, json " + + "FROM entity_extension " + + "WHERE id IN () AND extension = :extension " + + "ORDER BY id, extension") + @RegisterRowMapper(ExtensionRecordWithIdMapper.class) + List getExtensionBatch( + @BindList("ids") List ids, @Bind("extension") String extension); + + @SqlQuery( + "SELECT id, extension, json, jsonschema " + + "FROM entity_extension " + + "WHERE extension LIKE :extension " + + "ORDER BY id, extension") + @RegisterRowMapper(ExtensionWithIdAndSchemaRowMapper.class) + List getExtensionsByPrefixBatch( + @BindConcat( + value = "extension", + parts = {":extensionPrefix", "%"}) + String extensionPrefix); + + @Transaction + @ConnectionAwareSqlBatch( + value = + "INSERT INTO entity_extension (id, extension, json, jsonschema) " + + "VALUES (:id, :extension, :json, :jsonschema) " + + "ON DUPLICATE KEY UPDATE json = VALUES(json), jsonschema = VALUES(jsonschema)", + connectionType = MYSQL) + @ConnectionAwareSqlBatch( + value = + "INSERT INTO entity_extension (id, extension, json,jsonschema) VALUES (:id, :extension, :json::jsonb,:jsonschema) " + + "ON CONFLICT (id, extension) DO UPDATE SET json = EXCLUDED.json , jsonschema = EXCLUDED.jsonschema", + connectionType = POSTGRES) + void bulkUpsertExtensions( + @BindBean List extensionWithIdObjects); + + @RegisterRowMapper(ExtensionMapper.class) + @SqlQuery( + "SELECT extension, json FROM entity_extension WHERE id = :id AND extension " + + "LIKE CONCAT (:extensionPrefix, '.%') " + + "ORDER BY extension") + List getExtensions( + @BindUUID("id") UUID id, @Bind("extensionPrefix") String extensionPrefix); + + @RegisterRowMapper(ExtensionMapper.class) + @SqlQuery( + "SELECT extension, json FROM entity_extension WHERE id = :id AND jsonschema = :jsonSchema " + + "ORDER BY extension") + List getExtensionsByJsonSchema( + @BindUUID("id") UUID id, @Bind("jsonSchema") String jsonSchema); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT id, updatedAt, json FROM entity_extension " + + "WHERE updatedAt >= :startTs " + + "AND updatedAt <= :endTs " + + "AND jsonSchema = :entityType " + + "UNION " + + "SELECT id, updatedAt, json FROM

" + + "WHERE updatedAt >= :startTs AND " + + "updatedAt <= :endTs " + + ") combined WHERE 1=1 " + + " " + + "ORDER BY updatedAt DESC, id DESC " + + "LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT id, updatedAt, json FROM entity_extension " + + "WHERE updatedAt >= :startTs " + + "AND updatedAt <= :endTs " + + "AND jsonSchema = :entityType " + + "UNION " + + "SELECT id, updatedAt, json::jsonb FROM
" + + "WHERE updatedAt >= :startTs AND " + + "updatedAt <= :endTs " + + ") combined WHERE 1=1 " + + " " + + "ORDER BY updatedAt DESC, id DESC " + + "LIMIT :limit", + connectionType = POSTGRES) + @RegisterRowMapper(ExtensionMapper.class) + List getEntityHistoryByTimestampRange( + @Define("table") String table, + @Bind("startTs") long startTs, + @Bind("endTs") long endTs, + @Define("cursorCondition") String cursorCondition, + @Bind("entityType") String entityType, + @Bind("cursorUpdatedAt") Long cursorUpdatedAt, + @Bind("cursorId") String cursorId, + @Bind("limit") int limit); + + @SqlQuery( + value = + "SELECT SUM(cnt) FROM (" + + "SELECT COUNT(*) AS cnt FROM entity_extension " + + "WHERE updatedAt >= :startTs " + + "AND updatedAt <= :endTs " + + "AND jsonSchema = :entityType " + + "UNION ALL " + + "SELECT COUNT(*) AS cnt FROM
" + + "WHERE updatedAt >= :startTs AND " + + "updatedAt <= :endTs" + + ") total_counts") + int getEntityHistoryByTimestampRangeCount( + @Define("table") String table, + @Bind("startTs") long startTs, + @Bind("endTs") long endTs, + @Bind("entityType") String entityType); + + @RegisterRowMapper(ExtensionMapper.class) + @SqlQuery( + "SELECT extension, json FROM entity_extension WHERE id = :id AND extension " + + "LIKE CONCAT (:extensionPrefix, '.%') " + + "ORDER BY extension DESC " + + "LIMIT :limit OFFSET :offset") + List getExtensionsWithOffset( + @BindUUID("id") UUID id, + @Bind("extensionPrefix") String extensionPrefix, + @Bind("limit") int limit, + @Bind("offset") int offset); + + @SqlUpdate("DELETE FROM entity_extension WHERE id = :id AND extension = :extension") + void delete(@BindUUID("id") UUID id, @Bind("extension") String extension); + + @SqlUpdate("DELETE FROM entity_extension WHERE extension = :extension") + void deleteExtension(@Bind("extension") String extension); + + @SqlUpdate("DELETE FROM entity_extension WHERE id = :id") + void deleteAll(@BindUUID("id") UUID id); + + @SqlUpdate("DELETE FROM entity_extension WHERE id IN ()") + void deleteAllBatch(@BindList("ids") List ids); + } + + class EntityVersionPair { + @Getter private final Double version; + @Getter private final String entityJson; + + public EntityVersionPair(ExtensionRecord extensionRecord) { + this.version = EntityUtil.getVersion(extensionRecord.extensionName()); + this.entityJson = extensionRecord.extensionJson(); + } + } + + record ExtensionRecord(String extensionName, String extensionJson) {} + + record ExtensionRecordWithId(UUID id, String extensionName, String extensionJson) {} + + class ExtensionMapper implements RowMapper { + @Override + public ExtensionRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new ExtensionRecord(rs.getString("extension"), rs.getString("json")); + } + } + + class ExtensionRecordWithIdMapper implements RowMapper { + @Override + public ExtensionRecordWithId map(ResultSet rs, StatementContext ctx) throws SQLException { + String id = rs.getString("id"); + String extensionName = rs.getString("extension"); + String extensionJson = rs.getString("json"); + return new ExtensionRecordWithId(UUID.fromString(id), extensionName, extensionJson); + } + } + + @Getter + @Setter + @Builder + class ExtensionWithIdAndSchemaObject { + private String id; + private String extension; + private String json; + private String jsonschema; + } + + class ExtensionWithIdAndSchemaRowMapper implements RowMapper { + @Override + public ExtensionWithIdAndSchemaObject map(ResultSet rs, StatementContext ctx) + throws SQLException { + String id = rs.getString("id"); + String extensionName = rs.getString("extension"); + String extensionJson = rs.getString("json"); + String jsonSchema = rs.getString("jsonschema"); + return new ExtensionWithIdAndSchemaObject(id, extensionName, extensionJson, jsonSchema); + } + } + + @Getter + @Builder + class EntityRelationshipRecord { + private UUID id; + private String type; + private String json; + } + + @Getter + @Builder + class EntityRelationshipCount { + private UUID id; + private Integer count; + } + + @Getter + @Builder + class RelationTypeUsageCount { + private String relationType; + private Integer count; + } + + @Getter + @Builder + class EntityRelationshipObject { + private String fromId; + private String toId; + private String fromEntity; + private String toEntity; + private int relation; + private String json; + private String jsonSchema; + } + + @Getter + @Builder + class ReportDataRow { + private String rowNum; + private ReportData reportData; + } + + @Getter + @Builder + class QueryList { + private String fqn; + private Query query; + } + + interface EntityRelationshipDAO { + default void insert(UUID fromId, UUID toId, String fromEntity, String toEntity, int relation) { + insert(fromId, toId, fromEntity, toEntity, relation, "", null); + } + + default void insert( + UUID fromId, UUID toId, String fromEntity, String toEntity, int relation, String json) { + insert(fromId, toId, fromEntity, toEntity, relation, "", json); + } + + default void bulkInsertToRelationship( + UUID fromId, List toIds, String fromEntity, String toEntity, int relation) { + + List insertToRelationship = + toIds.stream() + .map( + testCase -> + EntityRelationshipObject.builder() + .fromId(fromId.toString()) + .toId(testCase.toString()) + .fromEntity(fromEntity) + .toEntity(toEntity) + .relation(relation) + .build()) + .collect(Collectors.toList()); + + bulkInsertTo(insertToRelationship); + } + + default void bulkRemoveToRelationship( + UUID fromId, List toIds, String fromEntity, String toEntity, int relation) { + + List toIdsAsString = toIds.stream().map(UUID::toString).toList(); + bulkRemoveTo(fromId, toIdsAsString, fromEntity, toEntity, relation); + } + + default void bulkRemoveFromRelationship( + List fromIds, UUID toId, String fromEntity, String toEntity, int relation) { + + List fromIdsAsString = fromIds.stream().map(UUID::toString).toList(); + bulkRemoveFrom(fromIdsAsString, toId, fromEntity, toEntity, relation); + } + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, relationType, json) " + + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation, :relationType, :json) " + + "ON DUPLICATE KEY UPDATE json = :json", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, relationType, json) VALUES " + + "(:fromId, :toId, :fromEntity, :toEntity, :relation, :relationType, (:json :: jsonb)) " + + "ON CONFLICT (fromId, toId, relation, relationType) DO UPDATE SET json = EXCLUDED.json", + connectionType = POSTGRES) + void insert( + @BindUUID("fromId") UUID fromId, + @BindUUID("toId") UUID toId, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation, + @Bind("relationType") String relationType, + @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES ", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation) VALUES " + + "ON CONFLICT DO NOTHING", + connectionType = POSTGRES) + void bulkInsertTo( + @BindBeanList( + value = "values", + propertyNames = {"fromId", "toId", "fromEntity", "toEntity", "relation"}) + List values); + + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " + + "SELECT :fromId, t.id, :fromEntity, :toEntity, :relation " + + "FROM
t", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " + + "SELECT :fromId, t.id, :fromEntity, :toEntity, :relation " + + "FROM
t " + + "ON CONFLICT DO NOTHING", + connectionType = POSTGRES) + void bulkInsertAllToRelationship( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation, + @Define("table") String table); + + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " + + "SELECT :fromId, t.id, :fromEntity, :toEntity, :relation " + + "FROM
t " + + "WHERE t.id NOT IN ()", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO entity_relationship (fromId, toId, fromEntity, toEntity, relation) " + + "SELECT :fromId, t.id, :fromEntity, :toEntity, :relation " + + "FROM
t " + + "WHERE t.id NOT IN () " + + "ON CONFLICT DO NOTHING", + connectionType = POSTGRES) + void bulkInsertAllToRelationshipWithExclusions( + @BindList("exclusionIds") List excludedIds, + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation, + @Define("table") String table); + + @SqlUpdate( + value = + "DELETE FROM entity_relationship WHERE fromId = :fromId " + + "AND fromEntity = :fromEntity AND toId IN () " + + "AND toEntity = :toEntity AND relation = :relation") + void bulkRemoveTo( + @BindUUID("fromId") UUID fromId, + @BindList("toIds") List toIds, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + @SqlUpdate( + "DELETE FROM entity_relationship " + + "WHERE fromEntity = :fromEntity " + + "AND fromId IN () " + + "AND toEntity = :toEntity " + + "AND relation = :relation " + + "AND toId = :toId") + void bulkRemoveFrom( + @BindList("fromIds") List fromIds, + @BindUUID("toId") UUID toId, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + @SqlUpdate( + "UPDATE entity_relationship " + + "SET fromId = :newFromId " + + "WHERE fromId = :oldFromId " + + "AND fromEntity = :fromEntity " + + "AND toEntity = :toEntity " + + "AND relation = :relation " + + "AND toId IN ()") + void bulkUpdateFromId( + @BindUUID("oldFromId") UUID oldFromId, + @BindUUID("newFromId") UUID newFromId, + @BindList("toIds") List toIds, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + // + // Find to operations + // + @SqlQuery( + "SELECT toId, toEntity, json FROM entity_relationship " + + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation IN ()") + @RegisterRowMapper(ToRelationshipMapper.class) + List findTo( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @BindList("relation") List relation); + + @SqlQuery( + "SELECT * FROM entity_relationship er1 JOIN entity_relationship er2 ON er1.toId = er2.toId WHERE er1.relation = 10 AND er1.fromEntity = 'domain' AND er2.fromId = :fromId AND er2.fromEntity = :fromEntity AND er2.relation = 13") + @RegisterRowMapper(RelationshipObjectMapper.class) + List findDownstreamDomains( + @BindUUID("fromId") UUID fromId, @Bind("fromEntity") String fromEntity); + + @SqlQuery( + "SELECT * FROM entity_relationship er1 JOIN entity_relationship er2 ON er1.toId = er2.fromId WHERE er1.relation = 10 AND er1.fromEntity = 'domain' AND er2.toId = :toId AND er2.toEntity = :toEntity AND er2.relation = 13") + @RegisterRowMapper(RelationshipObjectMapper.class) + List findUpstreamDomains( + @BindUUID("toId") UUID toId, @Bind("toEntity") String toEntity); + + @SqlQuery( + "select count(*) from entity_relationship where fromId in (select toId from entity_relationship where fromId = :fromDomainId and fromEntity = 'domain' and relation = 10) AND toId in (select toId from entity_relationship where fromId = :toDomainId and fromEntity = 'domain' and relation = 10) and relation = 13") + Integer countDomainChildAssets( + @BindUUID("fromDomainId") UUID fromDomainId, @BindUUID("toDomainId") UUID toId); + + @SqlQuery( + "SELECT * FROM entity_relationship er1 JOIN entity_relationship er2 ON er1.toId = er2.toId WHERE er1.relation = 10 AND er1.fromEntity = 'dataProduct' AND er2.fromId = :fromId AND er2.fromEntity = :fromEntity AND er2.relation = 13") + @RegisterRowMapper(RelationshipObjectMapper.class) + List findDownstreamDataProducts( + @BindUUID("fromId") UUID fromId, @Bind("fromEntity") String fromEntity); + + @SqlQuery( + "SELECT * FROM entity_relationship er1 JOIN entity_relationship er2 ON er1.toId = er2.fromId WHERE er1.relation = 10 AND er1.fromEntity = 'dataProduct' AND er2.toId = :toId AND er2.toEntity = :toEntity AND er2.relation = 13") + @RegisterRowMapper(RelationshipObjectMapper.class) + List findUpstreamDataProducts( + @BindUUID("toId") UUID toId, @Bind("toEntity") String toEntity); + + @SqlQuery( + "select count(*) from entity_relationship where fromId in (select toId from entity_relationship where fromId = :fromDataProductId and fromEntity = 'dataProduct' and relation = 10) AND toId in (select toId from entity_relationship where fromId = :toDataProductId and fromEntity = 'dataProduct' and relation = 10) and relation = 13") + Integer countDataProductsChildAssets( + @BindUUID("fromDataProductId") UUID fromDataProductId, + @BindUUID("toDataProductId") UUID toDataProductId); + + default List findTo(UUID fromId, String fromEntity, int relation) { + return findTo(fromId, fromEntity, List.of(relation)); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE fromId IN () " + + "AND relation = :relation " + + "AND fromEntity = :fromEntityType " + + "AND toEntity = :toEntityType " + + "AND deleted = FALSE") + @UseRowMapper(RelationshipObjectMapper.class) + List findToBatch( + @BindList("fromIds") List fromIds, + @Bind("relation") int relation, + @Bind("fromEntityType") String fromEntityType, + @Bind("toEntityType") String toEntityType); + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE fromId IN () " + + "AND relation = :relation " + + "AND toEntity = :toEntityType " + + "AND deleted = FALSE") + @UseRowMapper(RelationshipObjectMapper.class) + List findToBatch( + @BindList("fromIds") List fromIds, + @Bind("relation") int relation, + @Bind("toEntityType") String toEntityType); + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE fromId IN () " + + "AND relation = :relation " + + "AND toEntity = :toEntityType " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findToBatchWithCondition( + @BindList("fromIds") List fromIds, + @Bind("relation") int relation, + @Bind("toEntityType") String toEntityType, + @Define("cond") String condition); + + default List findToBatch( + List fromIds, int relation, String toEntityType, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findToBatchWithCondition(fromIds, relation, toEntityType, condition); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE fromId IN () " + + "AND relation = :relation " + + "AND fromEntity = :fromEntityType " + + "AND toEntity = :toEntityType " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findToBatchWithCondition( + @BindList("fromIds") List fromIds, + @Bind("relation") int relation, + @Bind("fromEntityType") String fromEntityType, + @Bind("toEntityType") String toEntityType, + @Define("cond") String condition); + + default List findToBatch( + List fromIds, + String fromEntityType, + String toEntityType, + int relation, + Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findToBatchWithCondition(fromIds, relation, fromEntityType, toEntityType, condition); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE fromId IN () " + + "AND relation = :relation " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findToBatchAllTypesWithCondition( + @BindList("fromIds") List fromIds, + @Bind("relation") int relation, + @Define("cond") String condition); + + default List findToBatchAllTypes( + List fromIds, int relation, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findToBatchAllTypesWithCondition(fromIds, relation, condition); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE fromId IN () " + + "AND relation IN () " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findToBatchAllTypesWithRelationsCondition( + @BindList("fromIds") List fromIds, + @BindList("relations") List relations, + @Define("cond") String condition); + + default List findToBatchAllTypes( + List fromIds, List relations, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findToBatchAllTypesWithRelationsCondition(fromIds, relations, condition); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE fromId IN () " + + "AND fromEntity = :fromEntity " + + "AND relation IN () " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findToBatchWithRelationsAndCondition( + @BindList("fromIds") List fromIds, + @Bind("fromEntity") String fromEntity, + @BindList("relations") List relations, + @Define("cond") String condition); + + default List findToBatchWithRelations( + List fromIds, String fromEntity, List relations, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findToBatchWithRelationsAndCondition(fromIds, fromEntity, relations, condition); + } + + default List findToBatchWithRelations( + List fromIds, String fromEntity, List relations) { + return findToBatchWithRelations(fromIds, fromEntity, relations, Include.ALL); + } + + @SqlQuery( + "SELECT toId, toEntity, json FROM entity_relationship " + + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity") + @RegisterRowMapper(ToRelationshipMapper.class) + List findTo( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @Bind("relation") int relation, + @Bind("toEntity") String toEntity); + + @SqlQuery( + "SELECT toId FROM entity_relationship " + + "WHERE fromId = :fromId AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity") + @RegisterRowMapper(ToRelationshipMapper.class) + List findToIds( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @Bind("relation") int relation, + @Bind("toEntity") String toEntity); + + @SqlQuery( + "SELECT COUNT(*) FROM entity_relationship " + + "WHERE fromId = :fromId AND toId = :toId AND fromEntity = :fromEntity AND toEntity = :toEntity AND relation = :relation") + int existsRelationship( + @BindUUID("fromId") UUID fromId, + @BindUUID("toId") UUID toId, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT fromId, COUNT(toId) FROM entity_relationship " + + "WHERE fromId IN () AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity " + + "GROUP BY fromId") + @RegisterRowMapper(ToRelationshipCountMapper.class) + List countFindTo( + @BindList("fromIds") List fromIds, + @Bind("fromEntity") String fromEntity, + @Bind("relation") int relation, + @Bind("toEntity") String toEntity); + + @SqlQuery( + "SELECT toId, toEntity, json FROM entity_relationship " + + "WHERE fromEntity = :fromEntity AND toEntity = :toEntity AND relation = :relation") + @RegisterRowMapper(ToRelationshipMapper.class) + List findAllByEntityTypes( + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT CASE WHEN relationType = '' THEN 'relatedTo' ELSE relationType END AS relationType, " + + "COUNT(*) AS cnt FROM entity_relationship " + + "WHERE fromEntity = :fromEntity AND toEntity = :toEntity AND relation = :relation " + + "GROUP BY CASE WHEN relationType = '' THEN 'relatedTo' ELSE relationType END") + @RegisterRowMapper(RelationTypeUsageCountMapper.class) + List countByRelationType( + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT COUNT(toId) FROM entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " + + "AND relation IN ()") + @RegisterRowMapper(ToRelationshipMapper.class) + int countFindTo( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @BindList("relation") List relation); + + @SqlQuery( + "SELECT toId, toEntity, json FROM entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " + + "AND relation IN () ORDER BY toId LIMIT :limit OFFSET :offset") + @RegisterRowMapper(ToRelationshipMapper.class) + List findToWithOffset( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @BindList("relation") List relation, + @Bind("offset") int offset, + @Bind("limit") int limit); + + @ConnectionAwareSqlQuery( + value = + "SELECT toId, toEntity, json FROM entity_relationship " + + "WHERE (JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) =:fromId OR fromId = :fromId) AND relation = :relation " + + "ORDER BY toId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT toId, toEntity, json FROM entity_relationship " + + "WHERE (json->'pipeline'->>'id' =:fromId OR fromId = :fromId) AND relation = :relation " + + "ORDER BY toId", + connectionType = POSTGRES) + @RegisterRowMapper(ToRelationshipMapper.class) + List findToPipeline( + @BindUUID("fromId") UUID fromId, @Bind("relation") int relation); + + // + // Find from operations + // + @SqlQuery( + "SELECT fromId, fromEntity, json FROM entity_relationship " + + "WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation AND fromEntity = :fromEntity ") + @RegisterRowMapper(FromRelationshipMapper.class) + List findFrom( + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation, + @Bind("fromEntity") String fromEntity); + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE toId IN () " + + "AND relation = :relation " + + "AND deleted = FALSE") + @UseRowMapper(RelationshipObjectMapper.class) + List findFromBatch( + @BindList("toIds") List toIds, @Bind("relation") int relation); + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE toId IN () " + + "AND relation = :relation " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findFromBatchWithCondition( + @BindList("toIds") List toIds, + @Bind("relation") int relation, + @Define("cond") String condition); + + default List findFromBatch( + List toIds, int relation, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findFromBatchWithCondition(toIds, relation, condition); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE toId IN () " + + "AND relation = :relation " + + "AND fromEntity = :fromEntityType " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findFromBatchWithEntityTypeAndCondition( + @BindList("toIds") List toIds, + @Bind("relation") int relation, + @Bind("fromEntityType") String fromEntityType, + @Define("cond") String condition); + + default List findFromBatch( + List toIds, int relation, String fromEntityType, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findFromBatchWithEntityTypeAndCondition(toIds, relation, fromEntityType, condition); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE toId IN () " + + "AND toEntity = :toEntityType " + + "AND relation IN () " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findFromBatchWithRelationsAndCondition( + @BindList("toIds") List toIds, + @Bind("toEntityType") String toEntityType, + @BindList("relations") List relations, + @Define("cond") String condition); + + default List findFromBatchWithRelations( + List toIds, String toEntityType, List relations, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findFromBatchWithRelationsAndCondition(toIds, toEntityType, relations, condition); + } + + default List findFromBatchWithRelations( + List toIds, String toEntityType, List relations) { + return findFromBatchWithRelations(toIds, toEntityType, relations, Include.ALL); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE toId IN () " + + "AND relation = :relation " + + "AND fromEntity = :fromEntityType " + + "AND deleted = FALSE") + @UseRowMapper(RelationshipObjectMapper.class) + List findFromBatch( + @BindList("toIds") List toIds, + @Bind("relation") int relation, + @Bind("fromEntityType") String fromEntityType); + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE toId IN () " + + "AND relation = :relation " + + "AND toEntity = :toEntityType " + + "AND deleted = FALSE") + @UseRowMapper(RelationshipObjectMapper.class) + List findFromBatch( + @BindList("toIds") List toIds, + @Bind("toEntityType") String toEntityType, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT fromId, fromEntity, json FROM entity_relationship " + + "WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation") + @RegisterRowMapper(FromRelationshipMapper.class) + List findFrom( + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + // Fetch relationships for specific relation types (TO direction: others -> entity) + // Used for owners, followers, domains, dataProducts, reviewers + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE toId = :entityId AND toEntity = :entityType " + + "AND relation IN () " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findToRelationshipsForEntityWithCondition( + @BindUUID("entityId") UUID entityId, + @Bind("entityType") String entityType, + @BindList("relations") List relations, + @Define("cond") String condition); + + default List findToRelationshipsForEntity( + UUID entityId, String entityType, List relations, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findToRelationshipsForEntityWithCondition(entityId, entityType, relations, condition); + } + + default List findToRelationshipsForEntity( + UUID entityId, String entityType, List relations) { + return findToRelationshipsForEntity(entityId, entityType, relations, Include.ALL); + } + + // Fetch relationships for specific relation types (FROM direction: entity -> others) + // Used for children, experts + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE fromId = :entityId AND fromEntity = :entityType " + + "AND relation IN () " + + "") + @UseRowMapper(RelationshipObjectMapper.class) + List findFromRelationshipsForEntityWithCondition( + @BindUUID("entityId") UUID entityId, + @Bind("entityType") String entityType, + @BindList("relations") List relations, + @Define("cond") String condition); + + default List findFromRelationshipsForEntity( + UUID entityId, String entityType, List relations, Include include) { + String condition = ""; + if (include == null || include == Include.NON_DELETED) { + condition = "AND deleted = FALSE"; + } else if (include == Include.DELETED) { + condition = "AND deleted = TRUE"; + } + return findFromRelationshipsForEntityWithCondition( + entityId, entityType, relations, condition); + } + + default List findFromRelationshipsForEntity( + UUID entityId, String entityType, List relations) { + return findFromRelationshipsForEntity(entityId, entityType, relations, Include.ALL); + } + + @SqlQuery( + "SELECT fromId, toId, fromEntity, toEntity, relation, json, jsonSchema " + + "FROM entity_relationship " + + "WHERE toId IN () " + + "AND relation = :relation " + + "AND fromEntity = :fromEntityType " + + "AND toEntity = :toEntityType " + + "AND deleted = FALSE") + @UseRowMapper(RelationshipObjectMapper.class) + List findFromBatch( + @BindList("toIds") List toIds, + @Bind("relation") int relation, + @Bind("fromEntityType") String fromEntityType, + @Bind("toEntityType") String toEntityType); + + @ConnectionAwareSqlQuery( + value = + "SELECT fromId, fromEntity, json FROM entity_relationship " + + "WHERE (JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) = :toId OR toId = :toId) AND relation = :relation " + + "ORDER BY fromId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT fromId, fromEntity, json FROM entity_relationship " + + "WHERE (json->'pipeline'->>'id' = :toId OR toId = :toId) AND relation = :relation " + + "ORDER BY fromId", + connectionType = POSTGRES) + @RegisterRowMapper(FromRelationshipMapper.class) + List findFromPipeline( + @BindUUID("toId") UUID toId, @Bind("relation") int relation); + + @ConnectionAwareSqlQuery( + value = + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship " + + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.source')) = :source AND (toId = :toId AND toEntity = :toEntity) " + + "AND relation = :relation ORDER BY fromId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship " + + "WHERE json->>'source' = :source AND (toId = :toId AND toEntity = :toEntity) " + + "AND relation = :relation ORDER BY fromId", + connectionType = POSTGRES) + @RegisterRowMapper(RelationshipObjectMapper.class) + List findLineageBySource( + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("source") String source, + @Bind("relation") int relation); + + @ConnectionAwareSqlQuery( + value = + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship " + + "WHERE (JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) =:toId OR toId = :toId) AND relation = :relation " + + "AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.source')) = :source ORDER BY toId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship " + + "WHERE (json->'pipeline'->>'id' =:toId OR toId = :toId) AND relation = :relation " + + "AND json->>'source' = :source ORDER BY toId", + connectionType = POSTGRES) + @RegisterRowMapper(RelationshipObjectMapper.class) + List findLineageBySourcePipeline( + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("source") String source, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT count(*) FROM entity_relationship WHERE fromEntity = :fromEntity AND toEntity = :toEntity") + int findIfAnyRelationExist( + @Bind("fromEntity") String fromEntity, @Bind("toEntity") String toEntity); + + @SqlQuery( + "SELECT json FROM entity_relationship WHERE fromId = :fromId " + + " AND toId = :toId " + + " AND relation = :relation ") + String getRelation( + @BindUUID("fromId") UUID fromId, + @BindUUID("toId") UUID toId, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship WHERE fromId = :fromId " + + " AND toId = :toId " + + " AND relation = :relation ") + @RegisterRowMapper(RelationshipObjectMapper.class) + EntityRelationshipObject getRecord( + @BindUUID("fromId") UUID fromId, + @BindUUID("toId") UUID toId, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship where relation = :relation ORDER BY fromId, toId LIMIT :limit OFFSET :offset") + @RegisterRowMapper(RelationshipObjectMapper.class) + List getRecordWithOffset( + @Bind("relation") int relation, @Bind("offset") long offset, @Bind("limit") int limit); + + @SqlQuery( + "SELECT toId, toEntity, fromId, fromEntity, relation, json, jsonSchema FROM entity_relationship ORDER BY fromId, toId LIMIT :limit OFFSET :offset") + @RegisterRowMapper(RelationshipObjectMapper.class) + List getAllRelationshipsPaginated( + @Bind("offset") long offset, @Bind("limit") int limit); + + @SqlQuery("SELECT COUNT(*) FROM entity_relationship") + long getTotalRelationshipCount(); + + // + // Delete Operations + // + @SqlUpdate( + "DELETE from entity_relationship WHERE fromId = :fromId " + + "AND fromEntity = :fromEntity AND toId = :toId AND toEntity = :toEntity " + + "AND relation = :relation") + int delete( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + @SqlUpdate( + "DELETE FROM entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " + + "AND toId = :toId AND toEntity = :toEntity AND relation = :relation " + + "AND relationType = :relationType") + int deleteWithRelationType( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation, + @Bind("relationType") String relationType); + + // Delete all the entity relationship fromID --- relation --> entity of type toEntity + @SqlUpdate( + "DELETE from entity_relationship WHERE fromId = :fromId AND fromEntity = :fromEntity " + + "AND relation = :relation AND toEntity = :toEntity") + void deleteFrom( + @BindUUID("fromId") UUID fromId, + @Bind("fromEntity") String fromEntity, + @Bind("relation") int relation, + @Bind("toEntity") String toEntity); + + // Delete all the entity relationship toId <-- relation -- entity of type fromEntity + @SqlUpdate( + "DELETE from entity_relationship WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation " + + "AND fromEntity = :fromEntity") + void deleteTo( + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation, + @Bind("fromEntity") String fromEntity); + + @SqlUpdate( + "DELETE from entity_relationship WHERE toId = :toId AND toEntity = :toEntity AND relation = :relation") + void deleteTo( + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + @SqlUpdate( + "DELETE FROM entity_relationship WHERE toId IN () " + + "AND toEntity = :toEntity AND relation = :relation AND fromEntity = :fromEntity") + void deleteToMany( + @BindList("toIds") List toIds, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation, + @Bind("fromEntity") String fromEntity); + + @SqlUpdate( + "DELETE FROM entity_relationship WHERE toId IN () " + + "AND toEntity = :toEntity AND relation = :relation") + void deleteToMany( + @BindList("toIds") List toIds, + @Bind("toEntity") String toEntity, + @Bind("relation") int relation); + + @SqlUpdate( + "DELETE FROM entity_relationship WHERE fromId IN () " + + "AND fromEntity = :fromEntity AND relation = :relation AND toEntity = :toEntity") + void deleteFromMany( + @BindList("fromIds") List fromIds, + @Bind("fromEntity") String fromEntity, + @Bind("relation") int relation, + @Bind("toEntity") String toEntity); + + @SqlUpdate( + "DELETE FROM entity_relationship WHERE fromId IN () " + + "AND fromEntity = :fromEntity AND relation = :relation") + void deleteFromMany( + @BindList("fromIds") List fromIds, + @Bind("fromEntity") String fromEntity, + @Bind("relation") int relation); + + // Optimized deleteAll implementation that splits OR query for better performance + @Transaction + default void deleteAll(UUID id, String entity) { + // Split OR query into two separate deletes for better index usage + deleteAllFrom(id, entity); + deleteAllTo(id, entity); + } + + @SqlUpdate("DELETE FROM entity_relationship WHERE fromId = :id AND fromEntity = :entity") + void deleteAllFrom(@BindUUID("id") UUID id, @Bind("entity") String entity); + + @SqlUpdate("DELETE FROM entity_relationship WHERE toId = :id AND toEntity = :entity") + void deleteAllTo(@BindUUID("id") UUID id, @Bind("entity") String entity); + + // Batch deletion methods for improved performance + @Transaction + default void batchDeleteRelationships(List entityIds, String entityType) { + if (entityIds == null || entityIds.isEmpty()) { + return; + } + + // Process in chunks of 500 to avoid hitting database query limits + int batchSize = 500; + for (int i = 0; i < entityIds.size(); i += batchSize) { + int endIndex = Math.min(i + batchSize, entityIds.size()); + List batch = + entityIds.subList(i, endIndex).stream() + .map(UUID::toString) + .collect(Collectors.toList()); + + batchDeleteFrom(batch, entityType); + batchDeleteTo(batch, entityType); + } + } + + @SqlUpdate( + "DELETE FROM entity_relationship WHERE fromId IN () AND fromEntity = :entityType") + void batchDeleteFrom(@BindList("ids") List ids, @Bind("entityType") String entityType); + + @SqlUpdate("DELETE FROM entity_relationship WHERE toId IN () AND toEntity = :entityType") + void batchDeleteTo(@BindList("ids") List ids, @Bind("entityType") String entityType); + + @SqlUpdate( + "DELETE FROM entity_relationship " + + "WHERE (toId IN () AND toEntity = :entity) " + + " OR (fromId IN () AND fromEntity = :entity)") + void deleteAllByThreadIds(@BindList("ids") List ids, @Bind("entity") String entity); + + @SqlUpdate("DELETE from entity_relationship WHERE fromId = :id or toId = :id") + void deleteAllWithId(@BindUUID("id") UUID id); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM entity_relationship " + + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.source')) = :source AND toId = :toId AND toEntity = :toEntity " + + "AND relation = :relation", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM entity_relationship " + + "WHERE json->>'source' = :source AND (toId = :toId AND toEntity = :toEntity) " + + "AND relation = :relation", + connectionType = POSTGRES) + void deleteLineageBySource( + @BindUUID("toId") UUID toId, + @Bind("toEntity") String toEntity, + @Bind("source") String source, + @Bind("relation") int relation); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM entity_relationship " + + "WHERE (JSON_UNQUOTE(JSON_EXTRACT(json, '$.pipeline.id')) =:toId OR toId = :toId) AND relation = :relation " + + "AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.source')) = :source ORDER BY toId", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM entity_relationship " + + "WHERE (json->'pipeline'->>'id' =:toId OR toId = :toId) AND relation = :relation " + + "AND json->>'source' = :source", + connectionType = POSTGRES) + void deleteLineageBySourcePipeline( + @BindUUID("toId") UUID toId, @Bind("source") String source, @Bind("relation") int relation); + + class FromRelationshipMapper implements RowMapper { + @Override + public EntityRelationshipRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return EntityRelationshipRecord.builder() + .id(UUID.fromString(rs.getString("fromId"))) + .type(rs.getString("fromEntity")) + .json(rs.getString("json")) + .build(); + } + } + + class ToRelationshipMapper implements RowMapper { + @Override + public EntityRelationshipRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return EntityRelationshipRecord.builder() + .id(UUID.fromString(rs.getString("toId"))) + .type(rs.getString("toEntity")) + .json(rs.getString("json")) + .build(); + } + } + + class ToRelationshipCountMapper implements RowMapper { + @Override + public EntityRelationshipCount map(ResultSet rs, StatementContext ctx) throws SQLException { + return EntityRelationshipCount.builder() + .id(UUID.fromString(rs.getString(1))) + .count(rs.getInt(2)) + .build(); + } + } + + class RelationTypeUsageCountMapper implements RowMapper { + @Override + public RelationTypeUsageCount map(ResultSet rs, StatementContext ctx) throws SQLException { + return RelationTypeUsageCount.builder() + .relationType(rs.getString("relationType")) + .count(rs.getInt("cnt")) + .build(); + } + } + + class RelationshipObjectMapper implements RowMapper { + @Override + public EntityRelationshipObject map(ResultSet rs, StatementContext ctx) throws SQLException { + return EntityRelationshipObject.builder() + .fromId(rs.getString("fromId")) + .fromEntity(rs.getString("fromEntity")) + .toEntity(rs.getString("toEntity")) + .toId(rs.getString("toId")) + .relation(rs.getInt("relation")) + .json(rs.getString("json")) + .jsonSchema(rs.getString("jsonSchema")) + .build(); + } + } + } + + interface FieldRelationshipDAO { + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, json) " + + "VALUES (:fromFQNHash, :toFQNHash, :fromFQN, :toFQN, :fromType, :toType, :relation, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, json) " + + "VALUES (:fromFQNHash, :toFQNHash, :fromFQN, :toFQN, :fromType, :toType, :relation, (:json :: jsonb)) " + + "ON CONFLICT (fromFQNHash, toFQNHash, relation) DO NOTHING", + connectionType = POSTGRES) + void insert( + @BindFQN("fromFQNHash") String fromFQNHash, + @BindFQN("toFQNHash") String toFQNHash, + @Bind("fromFQN") String fromFQN, + @Bind("toFQN") String toFQN, + @Bind("fromType") String fromType, + @Bind("toType") String toType, + @Bind("relation") int relation, + @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, jsonSchema, json) " + + "VALUES (:fromFQNHash, :toFQNHash, :fromFQN, :toFQN, :fromType, :toType, :relation, :jsonSchema, :json) " + + "ON DUPLICATE KEY UPDATE json = :json", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, jsonSchema, json) " + + "VALUES (:fromFQNHash, :toFQNHash, :fromFQN, :toFQN, :fromType, :toType, :relation, :jsonSchema, (:json :: jsonb)) " + + "ON CONFLICT (fromFQNHash, toFQNHash, relation) DO UPDATE SET json = EXCLUDED.json", + connectionType = POSTGRES) + void upsert( + @BindFQN("fromFQNHash") String fromFQNHash, + @BindFQN("toFQNHash") String toFQNHash, + @Bind("fromFQN") String fromFQN, + @Bind("toFQN") String toFQN, + @Bind("fromType") String fromType, + @Bind("toType") String toType, + @Bind("relation") int relation, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json); + + @SqlQuery( + "SELECT json FROM field_relationship WHERE " + + "fromFQNHash = :fromFQNHash AND toFQNHash = :toFQNHash AND fromType = :fromType " + + "AND toType = :toType AND relation = :relation") + String find( + @BindFQN("fromFQNHash") String fromFQNHash, + @BindFQN("toFQNHash") String toFQNHash, + @Bind("fromType") String fromType, + @Bind("toType") String toType, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT fromFQN, fromType, json FROM field_relationship WHERE " + + "toFQNHash = :toFQNHash AND toType = :toType AND relation = :relation") + @RegisterRowMapper(FromFieldMapper.class) + List> findFrom( + @BindFQN("toFQNHash") String toFQNHash, + @Bind("toType") String toType, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT fromFQN, toFQN, json FROM field_relationship WHERE " + + "fromFQNHash LIKE :concatFqnPrefixHash AND fromType = :fromType AND toType = :toType " + + "AND relation = :relation") + @RegisterRowMapper(ToFieldMapper.class) + List> listToByPrefix( + @BindConcat( + value = "concatFqnPrefixHash", + parts = {":fqnPrefixHash", "%"}, + hash = true) + String fqnPrefixHash, + @Bind("fromType") String fromType, + @Bind("toType") String toType, + @Bind("relation") int relation); + + @Deprecated(since = "Release 1.1") + @SqlQuery( + "SELECT DISTINCT fromFQN, toFQN FROM field_relationship WHERE fromFQNHash = '' or fromFQNHash is null or toFQNHash = '' or toFQNHash is null LIMIT :limit") + @RegisterRowMapper(FieldRelationShipMapper.class) + List> migrationListDistinctWithOffset(@Bind("limit") int limit); + + @SqlQuery( + "SELECT fromFQN, toFQN, json FROM field_relationship WHERE " + + "fromFQNHash = :fqnHash AND fromType = :type AND toType = :otherType AND relation = :relation " + + "UNION " + + "SELECT toFQN, fromFQN, json FROM field_relationship WHERE " + + "toFQNHash = :fqnHash AND toType = :type AND fromType = :otherType AND relation = :relation") + @RegisterRowMapper(ToFieldMapper.class) + List> listBidirectional( + @BindFQN("fqnHash") String fqnHash, + @Bind("type") String type, + @Bind("otherType") String otherType, + @Bind("relation") int relation); + + @SqlQuery( + "SELECT fromFQN, toFQN, json FROM field_relationship WHERE " + + "fromFQNHash LIKE :concatFqnPrefixHash AND fromType = :type AND toType = :otherType AND relation = :relation " + + "UNION " + + "SELECT toFQN, fromFQN, json FROM field_relationship WHERE " + + "toFQNHash LIKE :concatFqnPrefixHash AND toType = :type AND fromType = :otherType AND relation = :relation") + @RegisterRowMapper(ToFieldMapper.class) + List> listBidirectionalByPrefix( + @BindConcat( + value = "concatFqnPrefixHash", + parts = {":fqnPrefixHash", "%"}, + hash = true) + String fqnPrefixHash, + @Bind("type") String type, + @Bind("otherType") String otherType, + @Bind("relation") int relation); + + default void deleteAllByPrefix(String fqn) { + String prefix = String.format("%s%s%%", FullyQualifiedName.buildHash(fqn), Entity.SEPARATOR); + String condition = "WHERE (toFQNHash LIKE :prefix OR fromFQNHash LIKE :prefix)"; + Map bindMap = new HashMap<>(); + bindMap.put("prefix", prefix); + deleteAllByPrefixInternal(condition, bindMap); + } + + default void deleteAllByPrefixes(List threadIds) { + for (String threadId : threadIds) { + deleteAllByPrefix(threadId); + } + } + + @SqlUpdate("DELETE from field_relationship ") + void deleteAllByPrefixInternal( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlUpdate( + "DELETE from field_relationship WHERE fromFQNHash = :fromFQNHash AND toFQNHash = :toFQNHash AND fromType = :fromType " + + "AND toType = :toType AND relation = :relation") + void delete( + @BindFQN("fromFQNHash") String fromFQNHash, + @BindFQN("toFQNHash") String toFQNHash, + @Bind("fromType") String fromType, + @Bind("toType") String toType, + @Bind("relation") int relation); + + default void renameByToFQN(String oldToFQN, String newToFQN) { + renameByToFQNInternal( + oldToFQN, + FullyQualifiedName.buildHash(oldToFQN), + newToFQN, + FullyQualifiedName.buildHash(newToFQN)); // First rename targetFQN from oldFQN to newFQN + renameByToFQNPrefix(oldToFQN, newToFQN); + // Rename all the targetFQN prefixes starting with the oldFQN to newFQN + } + + @SqlUpdate( + "Update field_relationship set toFQN = :newToFQN , toFQNHash = :newToFQNHash " + + "where fromtype = 'THREAD' AND relation='3' AND toFQN = :oldToFQN and toFQNHash =:oldToFQNHash ;") + void renameByToFQNInternal( + @Bind("oldToFQN") String oldToFQN, + @Bind("oldToFQNHash") String oldToFQNHash, + @Bind("newToFQN") String newToFQN, + @Bind("newToFQNHash") String newToFQNHash); + + default void renameByToFQNPrefix(String oldToFQNPrefix, String newToFQNPrefix) { + String update = + String.format( + "UPDATE field_relationship SET toFQN = REPLACE(toFQN, '%s.', '%s.') , toFQNHash = REPLACE(toFQNHash, '%s.', '%s.') where fromtype = 'THREAD' AND relation='3' AND toFQN like '%s.%%' and toFQNHash like '%s.%%' ", + escapeApostrophe(oldToFQNPrefix), + escapeApostrophe(newToFQNPrefix), + FullyQualifiedName.buildHash(oldToFQNPrefix), + FullyQualifiedName.buildHash(newToFQNPrefix), + escapeApostrophe(oldToFQNPrefix), + FullyQualifiedName.buildHash(oldToFQNPrefix)); + renameByToFQNPrefixInternal(update); + } + + @SqlUpdate("") + void renameByToFQNPrefixInternal(@Define("update") String update); + + class FromFieldMapper implements RowMapper> { + @Override + public Triple map(ResultSet rs, StatementContext ctx) + throws SQLException { + return Triple.of(rs.getString("fromFQN"), rs.getString("fromType"), rs.getString("json")); + } + } + + class ToFieldMapper implements RowMapper> { + @Override + public Triple map(ResultSet rs, StatementContext ctx) + throws SQLException { + return Triple.of(rs.getString("fromFQN"), rs.getString("toFQN"), rs.getString("json")); + } + } + + class FieldRelationShipMapper implements RowMapper> { + @Override + public Pair map(ResultSet rs, StatementContext ctx) throws SQLException { + return Pair.of(rs.getString("fromFQN"), rs.getString("toFQN")); + } + } + + @Getter + @Setter + class FieldRelationship { + private String fromFQNHash; + private String toFQNHash; + private String fromFQN; + private String toFQN; + private String fromType; + private String toType; + private int relation; + private String jsonSchema; + private String json; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataAssetServiceDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataAssetServiceDAOs.java new file mode 100644 index 000000000000..37acb31d7ce4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataAssetServiceDAOs.java @@ -0,0 +1,1052 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.ListFilter.escapeApostrophe; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.customizer.BindMap; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.openmetadata.schema.entity.data.Container; +import org.openmetadata.schema.entity.data.Dashboard; +import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.schema.entity.data.Directory; +import org.openmetadata.schema.entity.data.File; +import org.openmetadata.schema.entity.data.SearchIndex; +import org.openmetadata.schema.entity.data.Spreadsheet; +import org.openmetadata.schema.entity.data.Worksheet; +import org.openmetadata.schema.entity.services.ApiService; +import org.openmetadata.schema.entity.services.DashboardService; +import org.openmetadata.schema.entity.services.DatabaseService; +import org.openmetadata.schema.entity.services.DriveService; +import org.openmetadata.schema.entity.services.MetadataService; +import org.openmetadata.schema.entity.services.SearchService; +import org.openmetadata.schema.entity.services.SecurityService; +import org.openmetadata.schema.entity.services.StorageService; +import org.openmetadata.schema.entity.services.connections.TestConnectionDefinition; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.util.FullyQualifiedName; + +public interface DataAssetServiceDAOs { + @CreateSqlObject + DatabaseDAO databaseDAO(); + + @CreateSqlObject + DatabaseSchemaDAO databaseSchemaDAO(); + + @CreateSqlObject + DashboardDAO dashboardDAO(); + + @CreateSqlObject + SearchIndexDAO searchIndexDAO(); + + @CreateSqlObject + DatabaseServiceDAO dbServiceDAO(); + + @CreateSqlObject + MetadataServiceDAO metadataServiceDAO(); + + @CreateSqlObject + DashboardServiceDAO dashboardServiceDAO(); + + @CreateSqlObject + StorageServiceDAO storageServiceDAO(); + + @CreateSqlObject + SearchServiceDAO searchServiceDAO(); + + @CreateSqlObject + SecurityServiceDAO securityServiceDAO(); + + @CreateSqlObject + ApiServiceDAO apiServiceDAO(); + + @CreateSqlObject + DriveServiceDAO driveServiceDAO(); + + @CreateSqlObject + ContainerDAO containerDAO(); + + @CreateSqlObject + DirectoryDAO directoryDAO(); + + @CreateSqlObject + FileDAO fileDAO(); + + @CreateSqlObject + SpreadsheetDAO spreadsheetDAO(); + + @CreateSqlObject + WorksheetDAO worksheetDAO(); + + @CreateSqlObject + TestConnectionDefinitionDAO testConnectionDefinitionDAO(); + + interface DashboardDAO extends EntityDAO { + @Override + default String getTableName() { + return "dashboard_entity"; + } + + @Override + default Class getEntityClass() { + return Dashboard.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface DashboardServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "dashboard_service_entity"; + } + + @Override + default Class getEntityClass() { + return DashboardService.class; + } + } + + interface DatabaseDAO extends EntityDAO { + @Override + default String getTableName() { + return "database_entity"; + } + + @Override + default Class getEntityClass() { + return Database.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @ConnectionAwareSqlQuery( + value = + "select JSON_EXTRACT(json, '$.fullyQualifiedName') from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select json ->> 'fullyQualifiedName' from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')", + connectionType = POSTGRES) + List getBrokenDatabase(); + + @SqlUpdate( + value = + "delete from database_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseService' and toEntity = 'database')") + int removeDatabase(); + } + + interface DatabaseSchemaDAO extends EntityDAO { + @Override + default String getTableName() { + return "database_schema_entity"; + } + + @Override + default Class getEntityClass() { + return DatabaseSchema.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @ConnectionAwareSqlQuery( + value = + "select JSON_EXTRACT(json, '$.fullyQualifiedName') from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select json ->> 'fullyQualifiedName' from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')", + connectionType = POSTGRES) + List getBrokenDatabaseSchemas(); + + @SqlUpdate( + value = + "delete from database_schema_entity where id not in (select toId from entity_relationship where fromEntity = 'database' and toEntity = 'databaseSchema')") + int removeBrokenDatabaseSchemas(); + } + + interface DatabaseServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "dbservice_entity"; + } + + @Override + default Class getEntityClass() { + return DatabaseService.class; + } + } + + interface MetadataServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "metadata_service_entity"; + } + + @Override + default Class getEntityClass() { + return MetadataService.class; + } + } + + interface TestConnectionDefinitionDAO extends EntityDAO { + @Override + default String getTableName() { + return "test_connection_definition"; + } + + @Override + default Class getEntityClass() { + return TestConnectionDefinition.class; + } + } + + interface StorageServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "storage_service_entity"; + } + + @Override + default Class getEntityClass() { + return StorageService.class; + } + } + + interface ContainerDAO extends EntityDAO { + @Override + default String getTableName() { + return "storage_container_entity"; + } + + @Override + default Class getEntityClass() { + return Container.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + + // By default, root will be false. We won't filter the results then + if (!root) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + + // Distinct method name (listRootBefore) is required: a same-signature `listBefore` + // here would override EntityDAO's default `listBefore(String, Map, String, int, + // String, String)` and make every non-root list call also pick up the depth check, + // silently filtering out child containers from generic `?service=...` listings. + return listRootBefore( + getTableName(), rootListingParams(filter), condition, limit, beforeName, beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + + if (!root) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + + return listRootAfter( + getTableName(), rootListingParams(filter), condition, limit, afterName, afterId); + } + + @Override + default int listCount(ListFilter filter) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + + if (!root) { + return EntityDAO.super.listCount(filter); + } + + return listRootCount( + getTableName(), getNameHashColumn(), rootListingParams(filter), condition); + } + + /** + * Build the bind map the listRoot SQL expects. The depth predicate + * ({@code fqnHash NOT LIKE :serviceHashChild}) needs the {@code serviceHashChild} + * bind to be set on every call, but {@link ListFilter#getServiceCondition} only + * adds it when {@code ?service=} is present. For the {@code ?root=true} case + * without a service filter — "all root containers across all services" — + * we default the bind to {@code %.%.%}, which excludes any fqnHash with two or more + * separators (everything strictly below the immediate level). Index usage is naturally + * weaker here since the prefix LIKE is also absent, but no-service root listings are + * rare and the result is at most one row per service. + */ + private static java.util.Map rootListingParams(ListFilter filter) { + java.util.Map params = new java.util.HashMap<>(filter.getQueryParams()); + params.putIfAbsent("serviceHashChild", "%.%.%"); + return params; + } + + // Root-only listing (?root=true) returns containers that are direct children of the + // service — i.e. one segment below the service in the FQN tree. + // + // Earlier implementations relied on `entity_relationship` as the source of truth ("a + // container is a root iff no inbound CONTAINS edge exists"). That broke under two + // separate failure modes: + // 1. Connectors (and bulk imports) that create deeply-nested containers without + // writing the parent CONTAINS edge — those orphans satisfy "no inbound edge" and + // surface at the service root, even though their FQN is many segments deep. The + // breadcrumb UI (which reads the FQN) and the listing (which reads the relationship) + // disagreed about where the container lived. + // 2. The NOT EXISTS anti-join needed a composite (fromEntity, toEntity, relation, toId) + // index to be cheap; under pgjdbc generic plans the planner often chose the + // ORDER BY index instead, falling back to a full-table scan and making the count + // query 1-2s on a service with hundreds of thousands of containers. + // + // The FQN is the canonical hierarchy in OpenMetadata (it's set unconditionally at write + // time and is what the breadcrumb UI consumes). `fqnHash` is built by joining + // fixed-width MD5 segments with '.', so depth follows from the count of separators — + // a direct child of the service has a fqnHash matching `.<32hex>` and + // contains no further '.'. We express "not a direct child" as `fqnHash LIKE + // .%.%` and reject those rows. ListFilter.getFqnPrefixCondition binds + // both `:serviceHash` (already used by the prefix LIKE in ) and + // `:serviceHashChild` (the `.%.%` companion) so the SQL just plugs them in. + @SqlQuery( + value = + "SELECT json FROM (" + + "SELECT name, id, ce.json FROM
ce " + + " AND " + + "ce.fqnHash NOT LIKE :serviceHashChild AND " + + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " + + "ORDER BY name DESC, id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name, id") + List listRootBefore( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId); + + @SqlQuery( + value = + "SELECT ce.json FROM
ce " + + " AND " + + "ce.fqnHash NOT LIKE :serviceHashChild AND " + + "(name > :afterName OR (name = :afterName AND id > :afterId)) " + + "ORDER BY name, id " + + "LIMIT :limit") + List listRootAfter( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId); + + @ConnectionAwareSqlQuery( + value = + "SELECT count() FROM
ce " + + " AND ce.fqnHash NOT LIKE :serviceHashChild", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM
ce " + + " AND ce.fqnHash NOT LIKE :serviceHashChild", + connectionType = POSTGRES) + int listRootCount( + @Define("table") String table, + @Define("nameHashColumn") String nameHashColumn, + @BindMap Map params, + @Define("sqlCondition") String mysqlCond); + + /** + * Lightweight projection used by paginated children listings. Pulls only the columns the + * UI's children table needs (id, name, displayName, fqn, description) plus the soft-delete + * flag. Skips JSON deserialization of heavy fields like {@code dataModel}, {@code tags}, + * and {@code owners} which can each carry MBs of column-schema metadata for parquet + * containers. The service reference is restored separately by + * {@link ContainerRepository#fetchAndSetDefaultService(java.util.List)}. + */ + @ConnectionAwareSqlQuery( + value = + "SELECT id, name, " + + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.displayName')) AS displayName, " + + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.fullyQualifiedName')) AS fqn, " + + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.description')) AS description, " + + "deleted " + + "FROM storage_container_entity WHERE id IN ()", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT id, name, " + + "json->>'displayName' AS displayName, " + + "json->>'fullyQualifiedName' AS fqn, " + + "json->>'description' AS description, " + + "deleted " + + "FROM storage_container_entity WHERE id IN ()", + connectionType = POSTGRES) + @RegisterRowMapper(ContainerSummaryRowMapper.class) + List findContainerSummaryRows(@BindList("ids") List ids); + + default List findContainerSummariesByIds(List ids) { + if (ids == null || ids.isEmpty()) { + return List.of(); + } + List idStrings = ids.stream().map(UUID::toString).distinct().toList(); + int maxChunkSize = 30000; + if (idStrings.size() <= maxChunkSize) { + return findContainerSummaryRows(idStrings); + } + List all = new ArrayList<>(idStrings.size()); + for (int i = 0; i < idStrings.size(); i += maxChunkSize) { + List chunk = idStrings.subList(i, Math.min(i + maxChunkSize, idStrings.size())); + all.addAll(findContainerSummaryRows(chunk)); + } + return all; + } + + // FQN-based direct-children page. The two binds (`:parentHash` = '.%' and + // `:parentHashChild` = '.%.%') together select containers whose FQN is exactly one + // segment below the parent — same shape used by the root listing in listRootAfter, just + // without the cursor pagination. Returns the slim projection used by the children table + // UI; the caller restores the service reference separately. `:includeDeleted` is a + // tri-state: 'NON_DELETED' (default), 'DELETED', or 'ALL'. `:nameLike` is a LIKE pattern + // applied to LOWER(name); callers pass '%' for "no filter" or '%%' + // for a substring search. ESCAPE '!' is set explicitly so the same pattern semantics + // hold on MySQL (default escape is '\') and PostgreSQL (default has no escape char). + // '!' is preferred over '\' because the JDBI ColonPrefixSqlParser scans string literals + // to skip ':' bind markers inside them, and a literal {@code '\'} confuses the scanner + // (it treats the trailing backslash as an escape and consumes the closing quote), + // leaving a downstream {@code :includeDeleted} bind un-substituted and the prepared + // statement malformed. + @ConnectionAwareSqlQuery( + value = + "SELECT id, name, " + + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.displayName')) AS displayName, " + + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.fullyQualifiedName')) AS fqn, " + + "JSON_UNQUOTE(JSON_EXTRACT(json, '$.description')) AS description, " + + "deleted " + + "FROM storage_container_entity " + + "WHERE fqnHash LIKE :parentHash AND fqnHash NOT LIKE :parentHashChild " + + " AND LOWER(name) LIKE :nameLike ESCAPE '!' " + + " AND (:includeDeleted = 'ALL' " + + " OR (:includeDeleted = 'DELETED' AND deleted = TRUE) " + + " OR (:includeDeleted = 'NON_DELETED' AND deleted = FALSE)) " + + "ORDER BY name, id LIMIT :limit OFFSET :offset", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT id, name, " + + "json->>'displayName' AS displayName, " + + "json->>'fullyQualifiedName' AS fqn, " + + "json->>'description' AS description, " + + "deleted " + + "FROM storage_container_entity " + + "WHERE fqnHash LIKE :parentHash AND fqnHash NOT LIKE :parentHashChild " + + " AND LOWER(name) LIKE :nameLike ESCAPE '!' " + + " AND (:includeDeleted = 'ALL' " + + " OR (:includeDeleted = 'DELETED' AND deleted = TRUE) " + + " OR (:includeDeleted = 'NON_DELETED' AND deleted = FALSE)) " + + "ORDER BY name, id LIMIT :limit OFFSET :offset", + connectionType = POSTGRES) + @RegisterRowMapper(ContainerSummaryRowMapper.class) + List listDirectChildSummariesByParentHash( + @Bind("parentHash") String parentHash, + @Bind("parentHashChild") String parentHashChild, + @Bind("nameLike") String nameLike, + @Bind("includeDeleted") String includeDeleted, + @Bind("limit") int limit, + @Bind("offset") int offset); + + @ConnectionAwareSqlQuery( + value = + "SELECT count(fqnHash) FROM storage_container_entity " + + "WHERE fqnHash LIKE :parentHash AND fqnHash NOT LIKE :parentHashChild " + + " AND LOWER(name) LIKE :nameLike ESCAPE '!' " + + " AND (:includeDeleted = 'ALL' " + + " OR (:includeDeleted = 'DELETED' AND deleted = TRUE) " + + " OR (:includeDeleted = 'NON_DELETED' AND deleted = FALSE))", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM storage_container_entity " + + "WHERE fqnHash LIKE :parentHash AND fqnHash NOT LIKE :parentHashChild " + + " AND LOWER(name) LIKE :nameLike ESCAPE '!' " + + " AND (:includeDeleted = 'ALL' " + + " OR (:includeDeleted = 'DELETED' AND deleted = TRUE) " + + " OR (:includeDeleted = 'NON_DELETED' AND deleted = FALSE))", + connectionType = POSTGRES) + int countDirectChildrenByParentHash( + @Bind("parentHash") String parentHash, + @Bind("parentHashChild") String parentHashChild, + @Bind("nameLike") String nameLike, + @Bind("includeDeleted") String includeDeleted); + + /** + * Cascade an FQN rename to every descendant container row when a parent is reassigned + * (#24294). The generic {@link EntityDAO#updateFqn(String, String)} only rewrites the + * top-level {@code $.fullyQualifiedName} via MySQL {@code JSON_REPLACE}, which leaves + * nested column FQNs ({@code $.dataModel.columns[*].fullyQualifiedName}) pointing at the + * old parent — silently breaking column lookups on MySQL. Postgres works only by accident + * because the base impl does a global {@code REPLACE(json::text, ...)}. + * + *

This override rewrites every {@code "fullyQualifiedName": "OLD_PREFIX..."} occurrence + * in the JSON document so column FQNs follow their container. {@code WHERE fqnHash LIKE + * 'oldHash.%'} restricts the update to descendants — the moved row itself updates via the + * standard {@code storeEntity} path after {@code setFullyQualifiedName} runs in memory. + * + *

On SQL interpolation: mirrors the pre-existing {@link EntityDAO#updateFqn} + * pattern — values are spliced into the SQL via {@link String#format} because the + * connection-aware {@code @SqlUpdate} dispatcher takes the full statement as a single + * {@code }/{@code } bind. The values come from server-side + * code (the FQN computed by {@code setFullyQualifiedName}, not user-supplied input), and + * {@link ListFilter#escapeApostrophe} handles the only SQL meta-character that can appear + * in a validated entity name. If a future code path lets arbitrary strings reach this + * method, swap to a parameterised form with {@code @Bind} parameters. + */ + @Override + default void updateFqn(String oldPrefix, String newPrefix) { + if (!getNameHashColumn().equals("fqnHash")) { + return; + } + String oldHash = FullyQualifiedName.buildHash(oldPrefix); + String newHash = FullyQualifiedName.buildHash(newPrefix); + String mySqlUpdate = + String.format( + "UPDATE %s SET json = CAST(REPLACE(CAST(json AS CHAR), " + + "'\"fullyQualifiedName\": \"%s.', '\"fullyQualifiedName\": \"%s.') AS JSON), " + + "fqnHash = REPLACE(fqnHash, '%s.', '%s.') " + + "WHERE fqnHash LIKE '%s.%%'", + getTableName(), + escapeApostrophe(oldPrefix), + escapeApostrophe(newPrefix), + oldHash, + newHash, + oldHash); + String postgresUpdate = + String.format( + "UPDATE %s SET json = REPLACE(json::text, " + + "'\"fullyQualifiedName\": \"%s.', '\"fullyQualifiedName\": \"%s.')::jsonb, " + + "fqnHash = REPLACE(fqnHash, '%s.', '%s.') " + + "WHERE fqnHash LIKE '%s.%%'", + getTableName(), + escapeApostrophe(oldPrefix), + escapeApostrophe(newPrefix), + oldHash, + newHash, + oldHash); + updateFqnInternal(mySqlUpdate, postgresUpdate); + } + + /** + * Cheap descendant count used by the PATCH re-parent guard (#24294) to short-circuit + * absurd subtree moves before any cascade work runs. {@code fqnHash LIKE 'oldHash.%'} + * matches every descendant row (excluding the moved container itself) and the index on + * {@code fqnHash} makes this an O(log n) lookup. + */ + @SqlQuery("SELECT COUNT(*) FROM storage_container_entity WHERE fqnHash LIKE :prefixLike") + int countDescendantsByPrefix(@Bind("prefixLike") String prefixLike); + } + + class ContainerSummaryRowMapper implements RowMapper { + @Override + public Container map(ResultSet rs, StatementContext ctx) throws SQLException { + return new Container() + .withId(UUID.fromString(rs.getString("id"))) + .withName(rs.getString("name")) + .withDisplayName(rs.getString("displayName")) + .withFullyQualifiedName(rs.getString("fqn")) + .withDescription(rs.getString("description")) + .withDeleted(rs.getBoolean("deleted")); + } + } + + interface SearchServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "search_service_entity"; + } + + @Override + default Class getEntityClass() { + return SearchService.class; + } + } + + interface SecurityServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "security_service_entity"; + } + + @Override + default Class getEntityClass() { + return SecurityService.class; + } + } + + interface ApiServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "api_service_entity"; + } + + @Override + default Class getEntityClass() { + return ApiService.class; + } + } + + interface DriveServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "drive_service_entity"; + } + + @Override + default Class getEntityClass() { + return DriveService.class; + } + } + + interface DirectoryDAO extends EntityDAO { + @Override + default String getTableName() { + return "directory_entity"; + } + + @Override + default Class getEntityClass() { + return Directory.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listBefore( + getTableName(), filter.getQueryParams(), sqlCondition, limit, beforeName, beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listAfter( + getTableName(), filter.getQueryParams(), sqlCondition, limit, afterName, afterId); + } + + @Override + default int listCount(ListFilter filter) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listCount(filter); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), sqlCondition); + } + + @SqlQuery( + value = + "SELECT json FROM (" + + "SELECT name,id, ce.json FROM

ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity = 'directory' AND toEntity = 'directory' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + " AND " + + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " + + "ORDER BY name DESC,id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id") + List listBefore( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId); + + @SqlQuery( + value = + "SELECT ce.json FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity = 'directory' AND toEntity = 'directory' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + " AND " + + "(name > :afterName OR (name = :afterName AND id > :afterId)) " + + "ORDER BY name,id " + + "LIMIT :limit") + List listAfter( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId); + + @ConnectionAwareSqlQuery( + value = + "SELECT count() FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity = 'directory' AND toEntity = 'directory' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + "", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity = 'directory' AND toEntity = 'directory' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + "", + connectionType = POSTGRES) + int listCount( + @Define("table") String table, + @Define("nameHashColumn") String nameHashColumn, + @BindMap Map params, + @Define("sqlCondition") String mysqlCond); + } + + interface FileDAO extends EntityDAO { + @Override + default String getTableName() { + return "file_entity"; + } + + @Override + default Class getEntityClass() { + return File.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listBefore( + getTableName(), filter.getQueryParams(), sqlCondition, limit, beforeName, beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listAfter( + getTableName(), filter.getQueryParams(), sqlCondition, limit, afterName, afterId); + } + + @Override + default int listCount(ListFilter filter) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listCount(filter); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), sqlCondition); + } + + @SqlQuery( + value = + "SELECT json FROM (" + + "SELECT name,id, ce.json FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity IN ('directory', 'spreadsheet') AND toEntity = 'file' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + " AND " + + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " + + "ORDER BY name DESC,id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id") + List listBefore( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId); + + @SqlQuery( + value = + "SELECT ce.json FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity IN ('directory', 'spreadsheet') AND toEntity = 'file' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + " AND " + + "(name > :afterName OR (name = :afterName AND id > :afterId)) " + + "ORDER BY name,id " + + "LIMIT :limit") + List listAfter( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId); + + @ConnectionAwareSqlQuery( + value = + "SELECT count() FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity IN ('directory', 'spreadsheet') AND toEntity = 'file' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + "", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity IN ('directory', 'spreadsheet') AND toEntity = 'file' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + "", + connectionType = POSTGRES) + int listCount( + @Define("table") String table, + @Define("nameHashColumn") String nameHashColumn, + @BindMap Map params, + @Define("sqlCondition") String mysqlCond); + } + + interface SpreadsheetDAO extends EntityDAO { + @Override + default String getTableName() { + return "spreadsheet_entity"; + } + + @Override + default Class getEntityClass() { + return Spreadsheet.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listBefore( + getTableName(), filter.getQueryParams(), sqlCondition, limit, beforeName, beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listAfter( + getTableName(), filter.getQueryParams(), sqlCondition, limit, afterName, afterId); + } + + @Override + default int listCount(ListFilter filter) { + boolean root = Boolean.parseBoolean(filter.getQueryParam("root")); + String condition = filter.getCondition(); + if (!root) { + return EntityDAO.super.listCount(filter); + } + String sqlCondition = String.format("%s AND er.toId is NULL", condition); + return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), sqlCondition); + } + + @SqlQuery( + value = + "SELECT json FROM (" + + "SELECT name,id, ce.json FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity = 'directory' AND toEntity = 'spreadsheet' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + " AND " + + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " + + "ORDER BY name DESC,id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id") + List listBefore( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId); + + @SqlQuery( + value = + "SELECT ce.json FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity = 'directory' AND toEntity = 'spreadsheet' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + " AND " + + "(name > :afterName OR (name = :afterName AND id > :afterId)) " + + "ORDER BY name,id " + + "LIMIT :limit") + List listAfter( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId); + + @ConnectionAwareSqlQuery( + value = + "SELECT count() FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity = 'directory' AND toEntity = 'spreadsheet' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + "", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM
ce " + + "LEFT JOIN (" + + " SELECT toId FROM entity_relationship " + + " WHERE fromEntity = 'directory' AND toEntity = 'spreadsheet' AND relation = 0 " + + ") er " + + "on ce.id = er.toId " + + "", + connectionType = POSTGRES) + int listCount( + @Define("table") String table, + @Define("nameHashColumn") String nameHashColumn, + @BindMap Map params, + @Define("sqlCondition") String mysqlCond); + } + + interface WorksheetDAO extends EntityDAO { + @Override + default String getTableName() { + return "worksheet_entity"; + } + + @Override + default Class getEntityClass() { + return Worksheet.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface SearchIndexDAO extends EntityDAO { + @Override + default String getTableName() { + return "search_index_entity"; + } + + @Override + default Class getEntityClass() { + return SearchIndex.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index 7ba78a50b78d..c972345f951a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -655,7 +655,8 @@ protected BulkOperationResult bulkAssetsOperation( // (FIELDS_STORED_AS_RELATIONSHIPS) and re-derived from entity_relationship on read. // Drop every cached variant of the asset so the next read rebuilds it from the // freshly-written relationships. - invalidateCacheForEntity(ref.getType(), ref.getId(), ref.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + ref.getType(), ref.getId(), ref.getFullyQualifiedName()); success.add(new BulkResponse().withRequest(ref)); result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); @@ -947,7 +948,7 @@ private void batchMigrateAssetDomains( // asset FQN from the relationship record's JSON so the by-name cache variant is evicted // too; otherwise GET-by-name would keep serving stale domain references until TTL. for (CollectionDAO.EntityRelationshipRecord record : assetRecords) { - invalidateCacheForReferencedEntity(record); + EntityCacheInvalidator.invalidateCacheForReferencedEntity(record); } } @@ -990,7 +991,7 @@ private void updateName(DataProduct updated) { .relationshipDAO() .findTo(updated.getId(), DATA_PRODUCT, Relationship.HAS.ordinal()); for (CollectionDAO.EntityRelationshipRecord record : assetRecords) { - invalidateCacheForReferencedEntity(record); + EntityCacheInvalidator.invalidateCacheForReferencedEntity(record); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index cead97e12816..a5f9ba38be72 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -316,7 +316,8 @@ protected BulkOperationResult bulkAssetsOperation( // the asset's cached entity bundle and the per-field domains/owners hash entry both // hold the previous-domain view. Drop every cached variant so the next read rebuilds // it from the freshly-written relationships. - invalidateCacheForEntity(ref.getType(), ref.getId(), ref.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + ref.getType(), ref.getId(), ref.getFullyQualifiedName()); success.add(new BulkResponse().withRequest(ref)); result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); @@ -589,11 +590,11 @@ private void updateName(Domain updated) { // Capture the descendants so the post-write pass can re-evict any entry a racing reader // re-populated with the pre-rename row between this call and the DAO updateFqn below. // The pass below runs after updateFqn but inside this transaction — see - // EntityRepository.invalidateCacheForRenameCascade for the residual pre-commit window. + // EntityCacheInvalidator.invalidateCacheForRenameCascade for the residual pre-commit window. List renamedDomains = - invalidateCacheForRenameCascade(Entity.DOMAIN, oldFqn); + EntityCacheInvalidator.invalidateCacheForRenameCascade(Entity.DOMAIN, oldFqn); List renamedDataProducts = - invalidateCacheForRenameCascade(Entity.DATA_PRODUCT, oldFqn); + EntityCacheInvalidator.invalidateCacheForRenameCascade(Entity.DATA_PRODUCT, oldFqn); // Update all child domains' FQNs and FQN hashes daoCollection.domainDAO().updateFqn(oldFqn, newFqn); @@ -615,8 +616,9 @@ private void updateName(Domain updated) { invalidateDomainReferencers(child.getId()); } - finishInvalidateCacheForRenameCascade(Entity.DOMAIN, renamedDomains); - finishInvalidateCacheForRenameCascade(Entity.DATA_PRODUCT, renamedDataProducts); + EntityCacheInvalidator.finishInvalidateCacheForRenameCascade(Entity.DOMAIN, renamedDomains); + EntityCacheInvalidator.finishInvalidateCacheForRenameCascade( + Entity.DATA_PRODUCT, renamedDataProducts); } private void invalidateDomainReferencers(UUID domainId) { @@ -628,7 +630,7 @@ private void invalidateDomainReferencers(UUID domainId) { .relationshipDAO() .findTo(domainId, Entity.DOMAIN, Relationship.HAS.ordinal()); for (CollectionDAO.EntityRelationshipRecord record : referencers) { - invalidateCacheForReferencedEntity(record); + EntityCacheInvalidator.invalidateCacheForReferencedEntity(record); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCacheInvalidator.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCacheInvalidator.java new file mode 100644 index 000000000000..927526f0ac81 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCacheInvalidator.java @@ -0,0 +1,475 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.function.IntSupplier; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.cache.CacheBundle; +import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.workflows.searchIndex.ReindexingUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Static entity-cache invalidation: per-entity, rename-cascade, referenced-entity and + * tag-based eviction, plus the transactional deferral scope. Extracted from EntityRepository. */ +public final class EntityCacheInvalidator { + private static final Logger LOG = LoggerFactory.getLogger(EntityCacheInvalidator.class); + + private EntityCacheInvalidator() {} + + /** + * Invalidate cache entries for every descendant of {@code oldPrefix} in the given entity type's + * DB table. Called by rename-cascade flows (e.g. DomainRepository.updateName) right before the + * bulk {@code UPDATE ... WHERE fqnHash LIKE 'oldPrefix.%'} so downstream reads don't see the + * stale (pre-rename) FQN on the children. + * + *

Publishes pub/sub for each descendant so peer OM instances drop their Guava entries too. + * + *

Returns the enumerated {@code (id, oldFqn)} pairs so the caller can pass them to {@link + * #finishInvalidateCacheForRenameCascade} once the rename-related DB statements have run — + * necessary because a reader landing in the window between this call and the bulk + * {@code UPDATE} can repopulate the by-id cache with the still-visible pre-rename row, and + * only a second invalidate pass after the DB statement can evict the poisoned entry. + * + *

Transactional scope: the existing rename call sites invoke both passes inside the + * same {@code @Transaction}-annotated updater, so the {@code finish} pass runs after the bulk + * {@code UPDATE} statement(s) but before the surrounding transaction commits. That + * closes the wide pre-update window (seconds, dominated by search-index walks) that CI + * traced as the failure mode, but a residual race remains: a concurrent reader landing + * between the {@code finish} pass and commit can still see the pre-rename row under + * READ COMMITTED and repopulate the cache. The window is on the order of milliseconds and + * we have no integration failures attributed to it; a true after-commit hook would close it + * fully and is tracked as a follow-up. + * + * @param entityType type name (e.g. {@code domain}, {@code dataProduct}, {@code tag}) + * @param oldPrefix fully qualified name prefix the rename is moving away from + */ + public static List invalidateCacheForRenameCascade( + String entityType, String oldPrefix) { + if (entityType == null || nullOrEmpty(oldPrefix)) { + return Collections.emptyList(); + } + EntityRepository repo; + try { + repo = Entity.getEntityRepository(entityType); + } catch (Exception e) { + return Collections.emptyList(); + } + if (repo == null || repo.getDao() == null) { + return Collections.emptyList(); + } + List affected; + try { + affected = repo.getDao().listDescendantIdFqnByPrefix(oldPrefix); + } catch (Exception e) { + LOG.warn( + "Failed to enumerate descendants for cache invalidation: type={} prefix={}", + entityType, + oldPrefix, + e); + return Collections.emptyList(); + } + if (affected.isEmpty()) { + return Collections.emptyList(); + } + dropDescendantCacheEntries(entityType, affected); + LOG.info( + "Invalidated cache for {} descendants of rename cascade: type={} prefix={}", + affected.size(), + entityType, + oldPrefix); + return affected; + } + + /** + * Post-rename-write pair to {@link #invalidateCacheForRenameCascade}. Re-evicts the cached + * forms of every descendant captured before the rename — by id and by the old FQN. Closes + * the wide race window where a concurrent reader arriving in the seconds between the + * pre-invalidate and the bulk rename {@code UPDATE} repopulates the by-id (or by-old-fqn) + * cache with the still-visible pre-rename row and pins that staleness for the entity TTL. + * + *

Called inside the same transaction as the rename writes (see {@link + * #invalidateCacheForRenameCascade} for the full transactional caveat); the millisecond + * window between this pass and commit is still racy but is not the failure mode CI traced. + * + *

Safe to call with an empty or null list (no-op). + */ + public static void finishInvalidateCacheForRenameCascade( + String entityType, List affected) { + if (entityType == null || affected == null || affected.isEmpty()) { + return; + } + dropDescendantCacheEntries(entityType, affected); + LOG.debug( + "Post-rename-write re-invalidated cache for {} descendants: type={}", + affected.size(), + entityType); + } + + private static void dropDescendantCacheEntries( + String entityType, List affected) { + // Rename cascades run inside the rename flush transaction. Route each descendant through + // invalidateCacheForEntity so the Guava-L1 eviction stays inline (cheap) while the Redis-L2 + // round trip is deferred to the post-commit drain when a flush scope is open — never issued + // while the pooled connection is held. The previous per-row pub/sub reason label was + // informational only (remote listeners evict L1 regardless), so the unified path is equivalent. + for (EntityDAO.EntityIdFqnPair row : affected) { + invalidateCacheForEntity(entityType, row.id, row.fqn); + } + } + + /** + * When a cache-deferral scope is open on the calling thread, {@link #invalidateCacheForEntity} + * records the Redis-L2 invalidation key here instead of issuing the (blocking, syncCommands) + * Redis round trip inline. The flush opens a scope before its DB transaction so no Redis call + * runs while a pooled connection is held — only the cheap local Guava-L1 invalidate stays inline + * — then drains the de-duplicated keys after commit, on the request thread, preserving + * read-your-write. {@code null} means "no scope active" and the Redis-L2 work runs inline. + */ + private static final ThreadLocal> + DEFERRED_CACHE_INVALIDATIONS = new ThreadLocal<>(); + + /** + * De-duplication key for a deferred Redis-L2 cache invalidation. Equality is on {@code + * (entityType, id)} only so repeated relationship writes touching the same entity collapse to one + * post-commit invalidation; the {@code fqn} is carried along (best non-null wins) so the by-name + * Redis variant is evicted when any caller knew the FQN. + */ + private static final class CacheInvalidationKey { + private final String entityType; + private final UUID id; + private final String fqn; + + private CacheInvalidationKey(String entityType, UUID id, String fqn) { + this.entityType = entityType; + this.id = id; + this.fqn = fqn; + } + + @Override + public boolean equals(Object other) { + boolean result = this == other; + if (!result && other instanceof CacheInvalidationKey key) { + result = entityType.equals(key.entityType) && id.equals(key.id); + } + return result; + } + + @Override + public int hashCode() { + return Objects.hash(entityType, id); + } + } + + /** + * Open a Redis-L2 cache-invalidation deferral scope on the current thread. While open, {@link + * #invalidateCacheForEntity} records keys instead of issuing the blocking Redis round trip. + * Returns {@code true} if this call opened the scope (caller owns draining/closing it), {@code + * false} if a scope was already open (nested call — the outer owner stays responsible). Pair a + * {@code true} result with {@link #drainCacheInvalidations()} after commit and {@link + * #clearCacheInvalidations()} on failure. + */ + static boolean beginCacheInvalidationDeferral() { + boolean opened = DEFERRED_CACHE_INVALIDATIONS.get() == null; + if (opened) { + DEFERRED_CACHE_INVALIDATIONS.set(new LinkedHashMap<>()); + } + return opened; + } + + /** Run every de-duplicated Redis-L2 invalidation captured since the scope opened, then close it. */ + static void drainCacheInvalidations() { + Map deferred = DEFERRED_CACHE_INVALIDATIONS.get(); + DEFERRED_CACHE_INVALIDATIONS.remove(); + if (deferred != null) { + for (CacheInvalidationKey key : deferred.values()) { + invalidateRedisL2ForEntity(key.entityType, key.id, key.fqn); + } + } + } + + /** Discard captured keys and close the scope without running them (failed transaction). */ + static void clearCacheInvalidations() { + DEFERRED_CACHE_INVALIDATIONS.remove(); + } + + /** + * Full local + cross-instance cache eviction for a single entity. Used by code paths that + * update a referring entity indirectly (e.g. data-product domain change updates the linked + * tables; a tag delete affects policies that embed it). Does the same work as + * {@link #invalidateCache(EntityInterface)} but doesn't require the full entity POJO — the + * {@code (type, id, fqn)} triple is enough to drop every cached variant. + * + *

The Guava-L1 eviction always runs inline (local, cheap). The Redis-L2 portion (base/by-name + * hash, relationship, bundle, lineage, pub/sub) issues a blocking {@code syncCommands} round + * trip, so when a deferral scope is active (a flush is holding a pooled DB connection) it is + * recorded and replayed post-commit on the request thread instead — never inside the handle. + */ + public static void invalidateCacheForEntity(String entityType, UUID id, String fqn) { + if (entityType == null || id == null) { + return; + } + // Guava L1 always cleared inline — it is a local map eviction, not a network round trip, and + // the rare uncached read path may have populated it even for UNCACHED_ENTITY_TYPES. + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); + if (fqn != null) { + EntityCaches.CACHE_WITH_NAME.invalidate(EntityCaches.cacheNameKey(entityType, fqn)); + } + // Skip every Redis op for entity types that are never cached. Bot/domain/data-product + // deletes cascade through many addRelationship/deleteRelationship calls; without this + // short-circuit each cascade pays for a pub/sub publish + multiple DELs that touch keys + // we never wrote — under heavy parallel load that pushes test budgets like + // TaskResourceIT.testDeletingBotCreatorCleansUpOpenSuggestionTasks past their 30 s window. + if (!EntityRepository.isCacheableEntityType(entityType)) { + return; + } + Map deferred = DEFERRED_CACHE_INVALIDATIONS.get(); + if (deferred != null) { + recordCacheInvalidation(deferred, entityType, id, fqn); + } else { + invalidateRedisL2ForEntity(entityType, id, fqn); + } + } + + private static void recordCacheInvalidation( + Map deferred, + String entityType, + UUID id, + String fqn) { + CacheInvalidationKey key = new CacheInvalidationKey(entityType, id, fqn); + CacheInvalidationKey existing = deferred.putIfAbsent(key, key); + if (existing != null && existing.fqn == null && fqn != null) { + deferred.put(key, key); + } + } + + private static void invalidateRedisL2ForEntity(String entityType, UUID id, String fqn) { + var cachedEntityDao = CacheBundle.getCachedEntityDao(); + if (cachedEntityDao != null) { + cachedEntityDao.invalidateBase(entityType, id); + if (fqn != null) { + cachedEntityDao.invalidateByName(entityType, fqn); + } + } + var cachedRelationshipDao = CacheBundle.getCachedRelationshipDao(); + if (cachedRelationshipDao != null) { + cachedRelationshipDao.invalidateOwners(entityType, id); + cachedRelationshipDao.invalidateDomains(entityType, id); + cachedRelationshipDao.invalidateContainer(entityType, id); + } + var cachedReadBundle = CacheBundle.getCachedReadBundle(); + if (cachedReadBundle != null) { + cachedReadBundle.invalidate(entityType, id); + } + var cachedLineage = CacheBundle.getCachedLineage(); + if (cachedLineage != null) { + cachedLineage.invalidate(id); + } + var pubsub = CacheBundle.getCacheInvalidationPubSub(); + if (pubsub != null) { + pubsub.publish(entityType, id, fqn, "ref-change"); + } + } + + /** + * Invalidate cache entries for an entity identified by an {@link + * CollectionDAO.EntityRelationshipRecord}. Extracts {@code fullyQualifiedName} from the record's + * JSON payload (when present) so the by-name cache variant is evicted alongside the by-id one. + * Callers that only have {@code (type, id)} and pass {@code fqn=null} leave GET-by-name entries + * stale until TTL expiry — use this when the referenced entity's FQN needs to be invalidated too. + */ + public static void invalidateCacheForReferencedEntity( + CollectionDAO.EntityRelationshipRecord record) { + if (record == null) { + return; + } + invalidateCacheForEntity(record.getType(), record.getId(), extractFqn(record.getJson())); + } + + /** + * Drop cached entity JSON, bundle, and relationship caches for every entity that carries the + * given tag FQN. The {@code tag_usage} table only stores {@code targetFQNHash}, so we cannot + * cheaply derive (type, id, fqn) from it; we lean on the search index instead — the same source + * the search-side {@code updateClassificationTagByFqnPrefix} reindex uses to find affected + * documents. Run this BEFORE the search reindex runs so the search query still matches documents + * by the old tag FQN. + * + *

Consistency tradeoff: coverage is bounded by search-index freshness. Entities + * tagged recently enough that the indexer hasn't picked them up are missed and fall back to + * the entity TTL (default 48h). On busy clusters with replication lag this can be minutes. + * If strict consistency is ever required, a direct {@code tag_usage} table query joined back + * to each candidate entity table would be more reliable at the cost of one round-trip per + * candidate type. + */ + public static int invalidateCacheForTaggedEntities(String tagFqn) { + int result = 0; + if (!nullOrEmpty(tagFqn)) { + result = + deferOrRunSearchBackedInvalidation( + () -> searchTaggedEntitiesAndInvalidate(tagFqn), tagFqn); + } + return result; + } + + /** + * Run the search-backed cache invalidation for {@code tagFqn} inline when no flush deferral scope + * is open, or capture it for post-commit drain when a rename/move cascade has opened one — the + * blocking ES search loop must never run while the DB transaction handle is held, and it must run + * exactly once even when a deadlock replays the cascade. The inline (non-flush) result is the live + * invalidated count; the deferred path returns {@code 0} because the work runs after this method + * returns and the count is only known then (it is logged inside {@code + * searchTaggedEntitiesAndInvalidate}). + */ + private static int deferOrRunSearchBackedInvalidation(IntSupplier invalidation, String tagFqn) { + int result = 0; + if (SearchRepository.isSearchWriteDeferralActive()) { + SearchRepository.deferOrRunSearchWrite( + invalidation::getAsInt, "invalidateCacheForTaggedEntities", null, tagFqn, null); + } else { + result = invalidation.getAsInt(); + } + return result; + } + + private static int searchTaggedEntitiesAndInvalidate(String tagFqn) { + int total = 0; + int from = 0; + boolean exhausted = false; + while (!exhausted) { + List page = findTaggedEntitiesPage(tagFqn, from); + if (page.isEmpty()) { + exhausted = true; + } else { + for (EntityReference ref : page) { + invalidateCacheForEntity(ref.getType(), ref.getId(), ref.getFullyQualifiedName()); + total++; + } + from += page.size(); + } + } + if (total > 0) { + LOG.info("Invalidated cache for {} entities tagged with: {}", total, tagFqn); + } + return total; + } + + private static List findTaggedEntitiesPage(String tagFqn, int from) { + List page; + try { + page = + ReindexingUtil.findReferenceInElasticSearchAcrossAllIndexes( + "tags.tagFQN", ReindexingUtil.escapeDoubleQuotes(tagFqn), from); + } catch (Exception e) { + LOG.warn("Search-based cache invalidation failed for tag={}", tagFqn, e); + page = List.of(); + } + return page; + } + + /** Bulk variant — invalidates entities tagged with any of the supplied tag FQNs. */ + public static int invalidateCacheForTaggedEntities(Collection tagFqns) { + int total = 0; + if (!nullOrEmpty(tagFqns)) { + for (String fqn : tagFqns) { + total += invalidateCacheForTaggedEntities(fqn); + } + if (total > 0) { + LOG.info( + "Invalidated cache for {} entities across {} renamed tag FQNs", total, tagFqns.size()); + } + } + return total; + } + + /** + * Convenience wrapper for tag-like entity renames (Tag, GlossaryTerm) where the rename cascades + * to descendants in the same entity table. Enumerates the descendant FQNs from the entity DAO + * BEFORE the DB rename rewrites them, then invalidates cached entities tagged with the prefix or + * any descendant. For Classification (where children live in a different entity table), enumerate + * child tag FQNs at the call site and pass them to {@link + * #invalidateCacheForTaggedEntities(Collection)} directly. + */ + public static int invalidateCacheForTaggedEntitiesAndDescendants( + String entityType, String oldPrefix) { + if (entityType == null || nullOrEmpty(oldPrefix)) { + return 0; + } + List fqns = new ArrayList<>(); + fqns.add(oldPrefix); + try { + EntityRepository repo = Entity.getEntityRepository(entityType); + if (repo != null && repo.getDao() != null) { + List descendants = + repo.getDao().listDescendantIdFqnByPrefix(oldPrefix); + for (EntityDAO.EntityIdFqnPair pair : descendants) { + if (pair.fqn != null && !pair.fqn.equals(oldPrefix)) { + fqns.add(pair.fqn); + } + } + } + } catch (Exception e) { + LOG.warn( + "Failed to enumerate descendants for tagged-entity invalidation: type={} fqn={}", + entityType, + oldPrefix, + e); + } + return invalidateCacheForTaggedEntities(fqns); + } + + private static String extractFqn(String json) { + if (json == null || json.isEmpty()) { + return null; + } + try { + var node = JsonUtils.readTree(json); + return node.hasNonNull("fullyQualifiedName") ? node.get("fullyQualifiedName").asText() : null; + } catch (Exception e) { + LOG.debug("Failed to extract fullyQualifiedName for cache invalidation", e); + return null; + } + } + + /** + * Invoked by {@link org.openmetadata.service.cache.CacheInvalidationPubSub} when another OM + * instance signals an entity change. Evicts this instance's per-process Guava caches so the next + * read pulls fresh data. Does not touch Redis — the writer already invalidated shared keys + * before publishing. + */ + public static void onRemoteCacheInvalidate(String entityType, UUID id, String fqn) { + if (entityType == null) { + return; + } + if (id != null) { + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); + } + if (fqn != null) { + EntityCaches.CACHE_WITH_NAME.invalidate(EntityCaches.cacheNameKey(entityType, fqn)); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCacheLoaders.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCacheLoaders.java new file mode 100644 index 000000000000..b23ce35d8a02 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCacheLoaders.java @@ -0,0 +1,199 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.schema.type.Include.ALL; + +import com.google.common.cache.CacheLoader; +import jakarta.validation.constraints.NotNull; +import java.util.Optional; +import java.util.UUID; +import lombok.NonNull; +import org.apache.commons.lang3.tuple.Pair; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.Entity; +import org.openmetadata.service.cache.CacheBundle; +import org.openmetadata.service.exception.EntityNotFoundException; + +// Guava CacheLoaders backing the entity name/id read-through caches. Extracted from +// EntityRepository; constructed by its buildEntityNameCache/buildEntityIdCache. +class EntityLoaderWithName extends CacheLoader, String> { + private static final org.slf4j.Logger LOG = + org.slf4j.LoggerFactory.getLogger(EntityLoaderWithName.class); + + @Override + public @NonNull String load(@NotNull Pair fqnPair) { + String entityType = fqnPair.getLeft(); + String fqn = fqnPair.getRight(); + EntityRepository repository = Entity.getEntityRepository(entityType); + EntityDAO dao = repository.getDao(); + + // Try to load from external cache first (read-through) for cacheable entity types. + if (EntityRepository.isCacheableEntityType(entityType)) { + var cachedEntityDao = CacheBundle.getCachedEntityDao(); + if (cachedEntityDao != null) { + Optional cachedJson = cachedEntityDao.getByName(entityType, fqn); + if (cachedJson.isPresent()) { + LOG.debug("CACHE HIT: Loading entity by name from Redis cache: {} {}", entityType, fqn); + try { + Class entityClass = repository.getEntityClass(); + EntityInterface entity = JsonUtils.readValue(cachedJson.get(), entityClass); + if (entity.getId() == null || entity.getFullyQualifiedName() == null) { + LOG.error( + "CACHE ERROR: Cached entity from name lookup is invalid! Evicting. Type: {}, Name: {}", + entityType, + fqn); + cachedEntityDao.deleteByName(entityType, fqn); + } else { + return cachedJson.get(); + } + } catch (Exception e) { + LOG.warn( + "Failed to deserialize cached entity, evicting and falling back to database: {} {}", + entityType, + fqn, + e); + try { + cachedEntityDao.deleteByName(entityType, fqn); + } catch (Exception evictError) { + LOG.debug( + "Failed to evict bad cache entry by name: {} {}", entityType, fqn, evictError); + } + } + } + LOG.debug("CACHE MISS: Entity not in Redis cache by name: {} {}", entityType, fqn); + } + } + + // Load raw JSON from database. User entities store nameHash off the lowercased FQN — + // UserDAO.findEntityByName lowercases the input. We call dao.findByName directly here + // to stay in the JSON-only path, so mirror the same case-fold for user types. + String lookupFqn = "user".equals(entityType) ? fqn.toLowerCase() : fqn; + LOG.debug("Loading entity by name from database: {} {}", entityType, lookupFqn); + String json = + dao.findByName( + dao.getTableName(), dao.getNameHashColumn(), lookupFqn, dao.getCondition(ALL)); + if (json == null) { + throw new EntityNotFoundException(String.format("Entity not found: %s %s", entityType, fqn)); + } + + // Validate + EntityInterface entity = JsonUtils.readValue(json, repository.getEntityClass()); + if (!EntityRepository.isValidEntityForCache(entity)) { + LOG.error( + "CRITICAL: Entity loaded from database by name is invalid! Type: {}, Name: {}, ID: {}", + entityType, + fqn, + entity == null ? "null" : entity.getId()); + throw new IllegalStateException( + String.format("Invalid entity from database: %s %s", entityType, fqn)); + } + + // Populate Redis on miss so subsequent reads (incl. cross-instance) can hit cache + if (EntityRepository.isCacheableEntityType(entityType)) { + var cachedEntityDao = CacheBundle.getCachedEntityDao(); + if (cachedEntityDao != null) { + try { + cachedEntityDao.putByName(entityType, fqn, json); + if (entity.getId() != null) { + cachedEntityDao.putBase(entityType, entity.getId(), json); + } + } catch (Exception e) { + LOG.debug("Failed to populate Redis on byName miss: {} {}", entityType, fqn, e); + } + } + } + + return json; + } +} + +class EntityLoaderWithId extends CacheLoader, String> { + private static final org.slf4j.Logger LOG = + org.slf4j.LoggerFactory.getLogger(EntityLoaderWithId.class); + + @Override + public @NonNull String load(@NotNull Pair idPair) { + String entityType = idPair.getLeft(); + UUID id = idPair.getRight(); + EntityRepository repository = Entity.getEntityRepository(entityType); + EntityDAO dao = repository.getDao(); + + // Try to load from external cache first (read-through) for cacheable entity types. + if (EntityRepository.isCacheableEntityType(entityType)) { + var cachedEntityDao = CacheBundle.getCachedEntityDao(); + if (cachedEntityDao != null) { + String cachedJson = cachedEntityDao.getBase(id, entityType); + if (cachedJson != null && !cachedJson.isEmpty()) { + LOG.debug("CACHE HIT: Loading entity from Redis cache: {} {}", entityType, id); + try { + Class entityClass = repository.getEntityClass(); + EntityInterface entity = JsonUtils.readValue(cachedJson, entityClass); + if (entity.getId() == null) { + LOG.error( + "CACHE ERROR: Cached entity has null ID! Evicting. Type: {}, Expected ID: {}", + entityType, + id); + cachedEntityDao.deleteBase(entityType, id); + } else { + return cachedJson; + } + } catch (Exception e) { + LOG.warn( + "Failed to deserialize cached entity, evicting and falling back to database: {} {}", + entityType, + id, + e); + try { + cachedEntityDao.deleteBase(entityType, id); + } catch (Exception evictError) { + LOG.debug("Failed to evict bad cache entry: {} {}", entityType, id, evictError); + } + } + } + LOG.debug("CACHE MISS: Entity not in Redis cache: {} {}", entityType, id); + } + } + + // Load raw JSON from database + LOG.debug("Loading entity from database: {} {}", entityType, id); + String json = dao.findById(dao.getTableName(), id, dao.getCondition(ALL)); + if (json == null) { + throw new EntityNotFoundException(String.format("Entity not found: %s %s", entityType, id)); + } + + // Validate + EntityInterface entity = JsonUtils.readValue(json, repository.getEntityClass()); + if (!EntityRepository.isValidEntityForCache(entity)) { + if (entity.getId() == null) { + LOG.error( + "CRITICAL: Entity loaded from database has null ID! Type: {}, Expected ID: {}, FQN: {}", + entityType, + id, + entity.getFullyQualifiedName()); + entity.setId(id); + json = JsonUtils.pojoToJson(entity); + } + entity = JsonUtils.readValue(json, repository.getEntityClass()); + if (!EntityRepository.isValidEntityForCache(entity)) { + LOG.error("Entity from database is invalid for caching: {} {}", entityType, id); + throw new IllegalStateException( + String.format("Invalid entity from database: %s %s", entityType, id)); + } + } + + return json; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCaches.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCaches.java new file mode 100644 index 000000000000..f978a72e616c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityCaches.java @@ -0,0 +1,156 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.cache.Weigher; +import java.util.Locale; +import java.util.UUID; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.openmetadata.service.Entity; +import org.openmetadata.service.config.CacheConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Static entity name/id read-through caches, count cache, and the field-fetch executor. + * Extracted from EntityRepository. EntityRepository keeps thin delegators for the public + * lifecycle methods (initCaches / setFieldFetchPoolSize / resetFieldFetchPoolSize). */ +public final class EntityCaches { + private static final Logger LOG = LoggerFactory.getLogger(EntityCaches.class); + + private EntityCaches() {} + + private static final int STRING_OBJECT_OVERHEAD_BYTES = 40; + + // Conservative upper-bound weight for a String: length() * 2 (UTF-16 worst-case) + 40 (header). + // On Java 21 with compact strings, LATIN1 content uses fewer bytes, so this overestimates + // slightly — which is intentional for memory capping. Zero allocation, single field read. + // Defaults used before CacheConfiguration is loaded at startup. initCaches() replaces these. + public static volatile LoadingCache, String> CACHE_WITH_NAME = + buildEntityNameCache( + CacheConfiguration.DEFAULT_ENTITY_CACHE_MAX_SIZE_BYTES, + CacheConfiguration.DEFAULT_ENTITY_CACHE_TTL_SECONDS); + public static volatile LoadingCache, String> CACHE_WITH_ID = + buildEntityIdCache( + CacheConfiguration.DEFAULT_ENTITY_CACHE_MAX_SIZE_BYTES, + CacheConfiguration.DEFAULT_ENTITY_CACHE_TTL_SECONDS); + + /** + * Canonical {@link #CACHE_WITH_NAME} key. User FQNs are lowercased at the DB layer + * ({@code UserDAO.findEntityByName}), so the Guava cache must use the same normalization — + * otherwise {@code Alice@x.com} and {@code alice@x.com} produce two split entries and + * invalidations written against the lowercased canonical form miss the mixed-case entry, + * serving stale data until TTL. + */ + public static Pair cacheNameKey(String entityType, String fqn) { + if (fqn != null && Entity.USER.equals(entityType)) { + return new ImmutablePair<>(entityType, fqn.toLowerCase(Locale.ROOT)); + } + return new ImmutablePair<>(entityType, fqn); + } + + /** + * Rebuild entity caches with values from {@link CacheConfiguration}. Called once during app + * startup after the configuration is loaded. Safe to call multiple times — subsequent calls + * replace the caches (old entries are lost, which is fine during initialization). + */ + public static void initCaches(CacheConfiguration config) { + CACHE_WITH_NAME = + buildEntityNameCache( + config.getEntityCacheMaxSizeBytes(), config.getEntityCacheTTLSeconds()); + CACHE_WITH_ID = + buildEntityIdCache(config.getEntityCacheMaxSizeBytes(), config.getEntityCacheTTLSeconds()); + LOG.info( + "Entity caches initialized: maxWeight={}MB, ttl={}s", + config.getEntityCacheMaxSizeBytes() / (1024 * 1024), + config.getEntityCacheTTLSeconds()); + } + + private static LoadingCache, String> buildEntityNameCache( + long maxWeightBytes, int ttlSeconds) { + return CacheBuilder.newBuilder() + .maximumWeight(maxWeightBytes) + .weigher( + (Weigher, String>) + (key, value) -> value.length() * 2 + STRING_OBJECT_OVERHEAD_BYTES) + .expireAfterWrite(ttlSeconds, TimeUnit.SECONDS) + .recordStats() + .build(new EntityLoaderWithName()); + } + + private static LoadingCache, String> buildEntityIdCache( + long maxWeightBytes, int ttlSeconds) { + return CacheBuilder.newBuilder() + .maximumWeight(maxWeightBytes) + .weigher( + (Weigher, String>) + (key, value) -> value.length() * 2 + STRING_OBJECT_OVERHEAD_BYTES) + .expireAfterWrite(ttlSeconds, TimeUnit.SECONDS) + .recordStats() + .build(new EntityLoaderWithId()); + } + + private static final int DEFAULT_FIELD_FETCH_POOL_SIZE = + Math.min(50, Runtime.getRuntime().availableProcessors() * 4); + private static final ThreadPoolExecutor FIELD_FETCH_EXECUTOR = + createFieldFetchExecutor(DEFAULT_FIELD_FETCH_POOL_SIZE); + + private static ThreadPoolExecutor createFieldFetchExecutor(int poolSize) { + ThreadPoolExecutor pool = + new ThreadPoolExecutor( + poolSize, + poolSize, + 60L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + java.lang.Thread.ofVirtual().name("om-field-fetch-", 0).factory()); + pool.allowCoreThreadTimeOut(true); + return pool; + } + + public static synchronized void setFieldFetchPoolSize(int size) { + int newSize = Math.max(1, Math.min(50, size)); + if (newSize <= FIELD_FETCH_EXECUTOR.getMaximumPoolSize()) { + FIELD_FETCH_EXECUTOR.setCorePoolSize(newSize); + FIELD_FETCH_EXECUTOR.setMaximumPoolSize(newSize); + } else { + FIELD_FETCH_EXECUTOR.setMaximumPoolSize(newSize); + FIELD_FETCH_EXECUTOR.setCorePoolSize(newSize); + } + LOG.info("Field-fetch pool resized to {} threads", newSize); + } + + public static synchronized void resetFieldFetchPoolSize() { + setFieldFetchPoolSize(DEFAULT_FIELD_FETCH_POOL_SIZE); + } + + public static final LoadingCache COUNT_CACHE = + CacheBuilder.newBuilder() + .maximumSize(500) + .expireAfterWrite(5, TimeUnit.MINUTES) + .recordStats() + .build( + new CacheLoader() { + @Override + public Integer load(String key) { + throw new UnsupportedOperationException("Use get() method with a custom loader"); + } + }); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDataDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDataDAOs.java new file mode 100644 index 000000000000..be424b518d7d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDataDAOs.java @@ -0,0 +1,1049 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.Relationship.CONTAINS; +import static org.openmetadata.schema.type.Relationship.MENTIONED_IN; +import static org.openmetadata.service.Entity.APPLICATION; +import static org.openmetadata.service.Entity.QUERY; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindMap; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.openmetadata.schema.entity.Bot; +import org.openmetadata.schema.entity.app.App; +import org.openmetadata.schema.entity.app.AppMarketPlaceDefinition; +import org.openmetadata.schema.entity.data.Chart; +import org.openmetadata.schema.entity.data.Glossary; +import org.openmetadata.schema.entity.data.GlossaryTerm; +import org.openmetadata.schema.entity.data.Metric; +import org.openmetadata.schema.entity.data.MlModel; +import org.openmetadata.schema.entity.data.Pipeline; +import org.openmetadata.schema.entity.data.Query; +import org.openmetadata.schema.entity.data.Report; +import org.openmetadata.schema.entity.data.StoredProcedure; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.policies.Policy; +import org.openmetadata.schema.entity.services.MessagingService; +import org.openmetadata.schema.entity.services.MlModelService; +import org.openmetadata.schema.entity.services.PipelineService; +import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.jdbi.BindConcat; + +public interface EntityDataDAOs { + @CreateSqlObject + TableDAO tableDAO(); + + @CreateSqlObject + QueryDAO queryDAO(); + + @CreateSqlObject + MetricDAO metricDAO(); + + @CreateSqlObject + ChartDAO chartDAO(); + + @CreateSqlObject + ApplicationDAO applicationDAO(); + + @CreateSqlObject + ApplicationMarketPlaceDAO applicationMarketPlaceDAO(); + + @CreateSqlObject + PipelineDAO pipelineDAO(); + + @CreateSqlObject + ReportDAO reportDAO(); + + @CreateSqlObject + MlModelDAO mlModelDAO(); + + @CreateSqlObject + GlossaryDAO glossaryDAO(); + + @CreateSqlObject + GlossaryTermDAO glossaryTermDAO(); + + @CreateSqlObject + BotDAO botDAO(); + + @CreateSqlObject + PolicyDAO policyDAO(); + + @CreateSqlObject + IngestionPipelineDAO ingestionPipelineDAO(); + + @CreateSqlObject + PipelineServiceDAO pipelineServiceDAO(); + + @CreateSqlObject + MlModelServiceDAO mlModelServiceDAO(); + + @CreateSqlObject + MessagingServiceDAO messagingServiceDAO(); + + @CreateSqlObject + StoredProcedureDAO storedProcedureDAO(); + + interface BotDAO extends EntityDAO { + @Override + default String getTableName() { + return "bot_entity"; + } + + @Override + default Class getEntityClass() { + return Bot.class; + } + } + + interface ChartDAO extends EntityDAO { + @Override + default String getTableName() { + return "chart_entity"; + } + + @Override + default Class getEntityClass() { + return Chart.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface ApplicationDAO extends EntityDAO { + @Override + default String getTableName() { + return "installed_apps"; + } + + @Override + default Class getEntityClass() { + return App.class; + } + + @ConnectionAwareSqlQuery( + value = + "SELECT id, name, JSON_UNQUOTE(JSON_EXTRACT(json, '$.displayName')) as displayName from installed_apps", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT id, name, json ->> 'displayName' as displayName from installed_apps", + connectionType = POSTGRES) + @RegisterRowMapper(AppEntityReferenceMapper.class) + List listAppsRef(); + + class AppEntityReferenceMapper implements RowMapper { + @Override + public EntityReference map(ResultSet rs, StatementContext ctx) throws SQLException { + String fqn = rs.getString("name"); + String displayName = rs.getString("displayName"); + + return new EntityReference() + .withId(UUID.fromString(rs.getString("id"))) + .withName(fqn) + .withDisplayName(displayName) + .withFullyQualifiedName(fqn) + .withType(APPLICATION); + } + } + } + + interface ApplicationMarketPlaceDAO extends EntityDAO { + @Override + default String getTableName() { + return "apps_marketplace"; + } + + @Override + default Class getEntityClass() { + return AppMarketPlaceDefinition.class; + } + } + + interface MessagingServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "messaging_service_entity"; + } + + @Override + default Class getEntityClass() { + return MessagingService.class; + } + } + + interface MetricDAO extends EntityDAO { + @Override + default String getTableName() { + return "metric_entity"; + } + + @Override + default Class getEntityClass() { + return Metric.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @ConnectionAwareSqlQuery( + value = + "SELECT DISTINCT customUnitOfMeasurement AS customUnit " + + "FROM metric_entity " + + "WHERE customUnitOfMeasurement IS NOT NULL " + + "AND customUnitOfMeasurement != '' " + + "AND deleted = false " + + "ORDER BY customUnitOfMeasurement", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT DISTINCT customUnitOfMeasurement AS customUnit " + + "FROM metric_entity " + + "WHERE customUnitOfMeasurement IS NOT NULL " + + "AND customUnitOfMeasurement != '' " + + "AND deleted = false " + + "ORDER BY customUnitOfMeasurement", + connectionType = POSTGRES) + List getDistinctCustomUnitsOfMeasurement(); + } + + interface MlModelDAO extends EntityDAO { + @Override + default String getTableName() { + return "ml_model_entity"; + } + + @Override + default Class getEntityClass() { + return MlModel.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface GlossaryDAO extends EntityDAO { + @Override + default String getTableName() { + return "glossary_entity"; + } + + @Override + default Class getEntityClass() { + return Glossary.class; + } + } + + interface GlossaryTermDAO extends EntityDAO { + @Override + default String getTableName() { + return "glossary_term_entity"; + } + + @Override + default Class getEntityClass() { + return GlossaryTerm.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default int listCount(ListFilter filter) { + String condition = filter.getCondition(); + String directChildrenOf = filter.getQueryParam("directChildrenOf"); + + if (!nullOrEmpty(directChildrenOf)) { + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; + } + + return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), condition); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String condition = filter.getCondition(); + String directChildrenOf = filter.getQueryParam("directChildrenOf"); + + if (!nullOrEmpty(directChildrenOf)) { + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; + } + + return listBefore( + getTableName(), filter.getQueryParams(), condition, limit, beforeName, beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String condition = filter.getCondition(); + String directChildrenOf = filter.getQueryParam("directChildrenOf"); + + if (!nullOrEmpty(directChildrenOf)) { + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; + } + return listAfter( + getTableName(), filter.getQueryParams(), condition, limit, afterName, afterId); + } + + @SqlQuery("select json FROM glossary_term_entity where fqnhash LIKE :concatFqnhash ") + List getNestedTerms( + @BindConcat( + value = "concatFqnhash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash); + + @SqlQuery("SELECT COUNT(*) FROM glossary_term_entity WHERE fqnHash LIKE :concatFqnhash ") + int countNestedTerms( + @BindConcat( + value = "concatFqnhash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash); + + @SqlQuery( + "SELECT COUNT(*) FROM glossary_term_entity WHERE fqnHash LIKE :glossaryHash AND LOWER(name) = LOWER(:termName)") + int getGlossaryTermCountIgnoreCase( + @BindConcat( + value = "glossaryHash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash, + @Bind("termName") String termName); + + @SqlQuery( + "SELECT COUNT(*) FROM glossary_term_entity WHERE fqnHash LIKE :glossaryHash AND LOWER(name) = LOWER(:termName) AND id != :excludeId") + int getGlossaryTermCountIgnoreCaseExcludingId( + @BindConcat( + value = "glossaryHash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash, + @Bind("termName") String termName, + @Bind("excludeId") String excludeId); + + @SqlQuery( + "SELECT json FROM glossary_term_entity WHERE fqnHash LIKE :glossaryHash AND LOWER(name) = LOWER(:termName)") + String getGlossaryTermByNameAndGlossaryIgnoreCase( + @BindConcat( + value = "glossaryHash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash, + @Bind("termName") String termName); + + // Search glossary terms by name and displayName using LIKE queries + // The displayName column is a generated column added in migration 1.9.3 + // entityStatus filtering uses generated column added in migration 1.12.2 + @SqlQuery( + "SELECT json FROM glossary_term_entity WHERE deleted = FALSE " + + "AND fqnHash LIKE :parentHash " + + "AND (LOWER(name) LIKE LOWER(:searchTerm) " + + "OR LOWER(COALESCE(displayName, '')) LIKE LOWER(:searchTerm)) " + + " " + + "ORDER BY name " + + "LIMIT :limit OFFSET :offset") + List searchGlossaryTerms( + @Bind("parentHash") String parentHash, + @Bind("searchTerm") String searchTerm, + @Define("statusCondition") String statusCondition, + @Bind("limit") int limit, + @Bind("offset") int offset); + } + + interface IngestionPipelineDAO extends EntityDAO { + @Override + default String getTableName() { + return "ingestion_pipeline_entity"; + } + + @Override + default Class getEntityClass() { + return IngestionPipeline.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default int listCount(ListFilter filter) { + String condition = + "INNER JOIN entity_relationship ON ingestion_pipeline_entity.id = entity_relationship.toId"; + + if (filter.getQueryParam("pipelineType") != null) { + String pipelineTypeCondition = + String.format(" and %s", filter.getPipelineTypeCondition(null)); + condition += pipelineTypeCondition; + } + + if (filter.getQueryParam("applicationType") != null) { + String applicationTypeCondition = + String.format(" and %s", filter.getApplicationTypeCondition()); + condition += applicationTypeCondition; + } + + if (filter.getQueryParam("service") != null) { + String serviceCondition = String.format(" and %s", filter.getServiceCondition(null)); + condition += serviceCondition; + } + + if (filter.getQueryParam("provider") != null) { + String providerCondition = + String.format(" and %s", filter.getProviderCondition(getTableName())); + condition += providerCondition; + } + + Map bindMap = new HashMap<>(); + String serviceType = filter.getQueryParam("serviceType"); + String provider = filter.getQueryParam("provider"); + if (!nullOrEmpty(provider)) { + bindMap.put("provider", provider); + } + if (!nullOrEmpty(serviceType)) { + + condition = + String.format( + "%s WHERE entity_relationship.fromEntity = :serviceType and entity_relationship.relation = :relation", + condition); + bindMap.put("relation", CONTAINS.ordinal()); + return listIngestionPipelineCount(condition, bindMap, filter.getQueryParams()); + } + return EntityDAO.super.listCount(filter); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String condition = + "INNER JOIN entity_relationship ON ingestion_pipeline_entity.id = entity_relationship.toId"; + + if (filter.getQueryParam("pipelineType") != null) { + String pipelineTypeCondition = + String.format(" and %s", filter.getPipelineTypeCondition(null)); + condition += pipelineTypeCondition; + } + + if (filter.getQueryParam("applicationType") != null) { + String applicationTypeCondition = + String.format(" and %s", filter.getApplicationTypeCondition()); + condition += applicationTypeCondition; + } + + if (filter.getQueryParam("service") != null) { + String serviceCondition = String.format(" and %s", filter.getServiceCondition(null)); + condition += serviceCondition; + } + + if (filter.getQueryParam("provider") != null) { + String providerCondition = + String.format(" and %s", filter.getProviderCondition(getTableName())); + condition += providerCondition; + } + + Map bindMap = new HashMap<>(); + String serviceType = filter.getQueryParam("serviceType"); + String provider = filter.getQueryParam("provider"); + if (!nullOrEmpty(provider)) { + bindMap.put("provider", provider); + } + if (!nullOrEmpty(serviceType)) { + + condition = + String.format( + "%s WHERE entity_relationship.fromEntity = :serviceType and entity_relationship.relation = :relation and (ingestion_pipeline_entity.name > :afterName OR (ingestion_pipeline_entity.name = :afterName AND ingestion_pipeline_entity.id > :afterId)) order by ingestion_pipeline_entity.name ASC,ingestion_pipeline_entity.id ASC LIMIT :limit", + condition); + + bindMap.put("relation", CONTAINS.ordinal()); + bindMap.put("afterName", afterName); + bindMap.put("afterId", afterId); + bindMap.put("limit", limit); + return listAfterIngestionPipelineByserviceType(condition, bindMap, filter.getQueryParams()); + } + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String condition = + "INNER JOIN entity_relationship ON ingestion_pipeline_entity.id = entity_relationship.toId"; + + if (filter.getQueryParam("pipelineType") != null) { + String pipelineTypeCondition = + String.format(" and %s", filter.getPipelineTypeCondition(null)); + condition += pipelineTypeCondition; + } + + if (filter.getQueryParam("applicationType") != null) { + String applicationTypeCondition = + String.format(" and %s", filter.getApplicationTypeCondition()); + condition += applicationTypeCondition; + } + + if (filter.getQueryParam("service") != null) { + String serviceCondition = String.format(" and %s", filter.getServiceCondition(null)); + condition += serviceCondition; + } + + if (filter.getQueryParam("provider") != null) { + String providerCondition = + String.format(" and %s", filter.getProviderCondition(getTableName())); + condition += providerCondition; + } + + Map bindMap = new HashMap<>(); + String serviceType = filter.getQueryParam("serviceType"); + String provider = filter.getQueryParam("provider"); + if (!nullOrEmpty(provider)) { + bindMap.put("provider", provider); + } + if (!nullOrEmpty(serviceType)) { + condition = + String.format( + "%s WHERE entity_relationship.fromEntity = :serviceType and entity_relationship.relation = :relation and (ingestion_pipeline_entity.name < :beforeName OR (ingestion_pipeline_entity.name = :beforeName AND ingestion_pipeline_entity.id < :beforeId)) order by ingestion_pipeline_entity.name DESC, ingestion_pipeline_entity.id DESC LIMIT :limit", + condition); + + bindMap.put("relation", CONTAINS.ordinal()); + bindMap.put("beforeName", beforeName); + bindMap.put("beforeId", beforeId); + bindMap.put("limit", limit); + return listBeforeIngestionPipelineByserviceType( + condition, bindMap, filter.getQueryParams()); + } + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + + @SqlQuery("SELECT ingestion_pipeline_entity.json FROM ingestion_pipeline_entity ") + List listAfterIngestionPipelineByserviceType( + @Define("cond") String cond, + @BindMap Map bindings, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM (SELECT ingestion_pipeline_entity.name, ingestion_pipeline_entity.id, ingestion_pipeline_entity.json FROM ingestion_pipeline_entity ) last_rows_subquery ORDER BY last_rows_subquery.name,last_rows_subquery.id") + List listBeforeIngestionPipelineByserviceType( + @Define("cond") String cond, + @BindMap Map bindings, + @BindMap Map params); + + @SqlQuery("SELECT count(*) FROM ingestion_pipeline_entity ") + int listIngestionPipelineCount( + @Define("cond") String cond, + @BindMap Map bindings, + @BindMap Map params); + } + + interface PipelineServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "pipeline_service_entity"; + } + + @Override + default Class getEntityClass() { + return PipelineService.class; + } + } + + interface MlModelServiceDAO extends EntityDAO { + @Override + default String getTableName() { + return "mlmodel_service_entity"; + } + + @Override + default Class getEntityClass() { + return MlModelService.class; + } + } + + interface PolicyDAO extends EntityDAO { + @Override + default String getTableName() { + return "policy_entity"; + } + + @Override + default Class getEntityClass() { + return Policy.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface ReportDAO extends EntityDAO { + @Override + default String getTableName() { + return "report_entity"; + } + + @Override + default Class getEntityClass() { + return Report.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface TableDAO extends EntityDAO

{ + @Override + default String getTableName() { + return "table_entity"; + } + + @Override + default Class
getEntityClass() { + return Table.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @ConnectionAwareSqlQuery( + value = + "select JSON_EXTRACT(json, '$.fullyQualifiedName') from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "select json ->> 'fullyQualifiedName' from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')", + connectionType = POSTGRES) + List getBrokenTables(); + + @SqlUpdate( + value = + "delete from table_entity where id not in (select toId from entity_relationship where fromEntity = 'databaseSchema' and toEntity = 'table')") + int removeBrokenTables(); + + @Override + default int listCount(ListFilter filter) { + String includeEmptyTestSuite = filter.getQueryParam("includeEmptyTestSuite"); + if (includeEmptyTestSuite != null && !Boolean.parseBoolean(includeEmptyTestSuite)) { + String condition = + String.format( + "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", + getTableName(), CONTAINS.ordinal(), Entity.TEST_SUITE); + String mySqlCondition = condition; + String postgresCondition = condition; + + mySqlCondition = + String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); + postgresCondition = + String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); + return listCount( + getTableName(), + getNameHashColumn(), + filter.getQueryParams(), + mySqlCondition, + postgresCondition); + } + + String condition = filter.getCondition(getTableName()); + return listCount( + getTableName(), getNameHashColumn(), filter.getQueryParams(), condition, condition); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String includeEmptyTestSuite = filter.getQueryParam("includeEmptyTestSuite"); + if (includeEmptyTestSuite != null && !Boolean.parseBoolean(includeEmptyTestSuite)) { + String condition = + String.format( + "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", + getTableName(), CONTAINS.ordinal(), Entity.TEST_SUITE); + String mySqlCondition = condition; + String postgresCondition = condition; + + mySqlCondition = + String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); + postgresCondition = + String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); + return listBefore( + getTableName(), + filter.getQueryParams(), + mySqlCondition, + postgresCondition, + limit, + beforeName, + beforeId); + } + String condition = filter.getCondition(getTableName()); + return listBefore( + getTableName(), + filter.getQueryParams(), + condition, + condition, + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String includeEmptyTestSuite = filter.getQueryParam("includeEmptyTestSuite"); + if (includeEmptyTestSuite != null && !Boolean.parseBoolean(includeEmptyTestSuite)) { + String condition = + String.format( + "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", + getTableName(), CONTAINS.ordinal(), Entity.TEST_SUITE); + String mySqlCondition = condition; + String postgresCondition = condition; + + mySqlCondition = + String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); + postgresCondition = + String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); + return listAfter( + getTableName(), + filter.getQueryParams(), + mySqlCondition, + postgresCondition, + limit, + afterName, + afterId); + } + String condition = filter.getCondition(getTableName()); + return listAfter( + getTableName(), filter.getQueryParams(), condition, condition, limit, afterName, afterId); + } + } + + interface StoredProcedureDAO extends EntityDAO { + @Override + default String getTableName() { + return "stored_procedure_entity"; + } + + @Override + default Class getEntityClass() { + return StoredProcedure.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface QueryDAO extends EntityDAO { + @Override + default String getTableName() { + return "query_entity"; + } + + @Override + default Class getEntityClass() { + return Query.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + + @Override + default int listCount(ListFilter filter) { + String entityId = filter.getQueryParam("entityId"); + String condition = + "INNER JOIN entity_relationship ON query_entity.id = entity_relationship.toId"; + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId)) { + condition = + String.format( + "%s WHERE entity_relationship.fromId = :id and entity_relationship.relation = :relation and entity_relationship.toEntity = :toEntityType", + condition); + bindMap.put("id", entityId); + bindMap.put("relation", MENTIONED_IN.ordinal()); + bindMap.put("toEntityType", QUERY); + return listQueryCount(condition, bindMap); + } + return EntityDAO.super.listCount(filter); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String entityId = filter.getQueryParam("entityId"); + String condition = + "INNER JOIN entity_relationship ON query_entity.id = entity_relationship.toId"; + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId)) { + condition = + String.format( + "%s WHERE entity_relationship.fromId = :entityId and entity_relationship.relation = :relation and entity_relationship.toEntity = :toEntity and (query_entity.name < :beforeName OR (query_entity.name = :beforeName AND query_entity.id < :beforeId)) order by query_entity.name DESC, query_entity.id DESC LIMIT :limit", + condition); + bindMap.put("entityId", entityId); + bindMap.put("relation", MENTIONED_IN.ordinal()); + bindMap.put("toEntity", QUERY); + bindMap.put("beforeName", beforeName); + bindMap.put("beforeId", beforeId); + bindMap.put("limit", limit); + return listBeforeQueriesByEntityId(condition, bindMap); + } + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String entityId = filter.getQueryParam("entityId"); + String condition = + "INNER JOIN entity_relationship ON query_entity.id = entity_relationship.toId"; + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId)) { + condition = + String.format( + "%s WHERE entity_relationship.fromId = :entityId and entity_relationship.relation = :relation and entity_relationship.toEntity = :toEntity and (query_entity.name > :afterName OR (query_entity.name = :afterName AND query_entity.name > :afterId)) order by query_entity.name ASC,query_entity.id ASC LIMIT :limit", + condition); + + bindMap.put("entityId", entityId); + bindMap.put("relation", MENTIONED_IN.ordinal()); + bindMap.put("toEntity", QUERY); + bindMap.put("afterName", afterName); + bindMap.put("afterId", afterId); + bindMap.put("limit", limit); + return listAfterQueriesByEntityId(condition, bindMap); + } + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + + @SqlQuery("SELECT query_entity.json FROM query_entity ") + List listAfterQueriesByEntityId( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery( + "SELECT json FROM (SELECT query_entity.name, query_entity.id, query_entity.json FROM query_entity ) last_rows_subquery ORDER BY name,id") + List listBeforeQueriesByEntityId( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery("SELECT count(*) FROM query_entity ") + int listQueryCount(@Define("cond") String cond, @BindMap Map bindings); + } + + interface PipelineDAO extends EntityDAO { + @Override + default String getTableName() { + return "pipeline_entity"; + } + + @Override + default Class getEntityClass() { + return Pipeline.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String status = filter.getQueryParam("status"); + if (status != null && !status.isEmpty()) { + // Remove status from filter to avoid SQL error + Map params = new HashMap<>(filter.getQueryParams()); + params.remove("status"); + ListFilter cleanFilter = new ListFilter(filter.getInclude()); + params.forEach(cleanFilter::addQueryParam); + + // Build condition with status JOIN + String condition = cleanFilter.getCondition(); + String statusCondition = + buildStatusJoinCondition(getTableName(), condition, status, beforeName, beforeId, true); + return listBeforeWithStatus( + statusCondition, getBindMap(cleanFilter, status, limit, beforeName, beforeId)); + } + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String status = filter.getQueryParam("status"); + if (status != null && !status.isEmpty()) { + // Remove status from filter to avoid SQL error + Map params = new HashMap<>(filter.getQueryParams()); + params.remove("status"); + ListFilter cleanFilter = new ListFilter(filter.getInclude()); + params.forEach(cleanFilter::addQueryParam); + + // Build condition with status JOIN + String condition = cleanFilter.getCondition(); + String statusCondition = + buildStatusJoinCondition(getTableName(), condition, status, afterName, afterId, false); + return listAfterWithStatus( + statusCondition, getBindMap(cleanFilter, status, limit, afterName, afterId)); + } + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + + @Override + default int listCount(ListFilter filter) { + String status = filter.getQueryParam("status"); + if (status != null && !status.isEmpty()) { + // Remove status from filter to avoid SQL error + Map params = new HashMap<>(filter.getQueryParams()); + params.remove("status"); + ListFilter cleanFilter = new ListFilter(filter.getInclude()); + params.forEach(cleanFilter::addQueryParam); + + // Build condition with status JOIN + String condition = cleanFilter.getCondition(); + String statusCondition = buildStatusCountCondition(getTableName(), condition, status); + return listCountWithStatus(statusCondition, getBindMap(cleanFilter, status, 0, null, null)); + } + return EntityDAO.super.listCount(filter); + } + + default String buildStatusJoinCondition( + String tableName, + String baseCondition, + String status, + String name, + String id, + boolean isBefore) { + String orderDirection = isBefore ? "DESC" : "ASC"; + String nameComparison = isBefore ? "<" : ">"; + String idComparison = isBefore ? "<" : ">"; + + return String.format( + "INNER JOIN (" + + " SELECT entityFQNHash, JSON_UNQUOTE(JSON_EXTRACT(json, '$.executionStatus')) as execStatus " + + " FROM entity_extension_time_series " + + " WHERE extension = 'pipeline.pipelineStatus' " + + " AND timestamp = (SELECT MAX(timestamp) FROM entity_extension_time_series eets2 " + + " WHERE eets2.entityFQNHash = entity_extension_time_series.entityFQNHash " + + " AND eets2.extension = 'pipeline.pipelineStatus') " + + ") latest_status ON %s.fqnHash = latest_status.entityFQNHash " + + "%s AND latest_status.execStatus = :status " + + "AND (%s.name %s :beforeAfterName OR (%s.name = :beforeAfterName AND %s.id %s :beforeAfterId)) " + + "ORDER BY %s.name %s, %s.id %s LIMIT :limit", + tableName, + baseCondition, + tableName, + nameComparison, + tableName, + tableName, + idComparison, + tableName, + orderDirection, + tableName, + orderDirection); + } + + default String buildStatusCountCondition( + String tableName, String baseCondition, String status) { + return String.format( + "INNER JOIN (" + + " SELECT entityFQNHash, JSON_UNQUOTE(JSON_EXTRACT(json, '$.executionStatus')) as execStatus " + + " FROM entity_extension_time_series " + + " WHERE extension = 'pipeline.pipelineStatus' " + + " AND timestamp = (SELECT MAX(timestamp) FROM entity_extension_time_series eets2 " + + " WHERE eets2.entityFQNHash = entity_extension_time_series.entityFQNHash " + + " AND eets2.extension = 'pipeline.pipelineStatus') " + + ") latest_status ON %s.fqnHash = latest_status.entityFQNHash " + + "%s AND latest_status.execStatus = :status", + tableName, baseCondition); + } + + default Map getBindMap( + ListFilter filter, String status, int limit, String name, String id) { + Map bindMap = new HashMap<>(); + if (status != null) { + bindMap.put("status", status); + } + if (limit > 0) { + bindMap.put("limit", limit); + } + if (name != null) { + bindMap.put("beforeAfterName", name); + } + if (id != null) { + bindMap.put("beforeAfterId", id); + } + // Add filter params + bindMap.putAll(filter.getQueryParams()); + return bindMap; + } + + @SqlQuery("SELECT json FROM pipeline_entity ") + List listAfterWithStatus( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery( + "SELECT json FROM (SELECT name, id, json FROM pipeline_entity ) last_rows_subquery ORDER BY name, id") + List listBeforeWithStatus( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery("SELECT count(*) FROM pipeline_entity ") + int listCountWithStatus(@Define("cond") String cond, @BindMap Map bindings); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRelationshipRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRelationshipRepository.java index 16acb2b27f4b..2e1c85a34c22 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRelationshipRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRelationshipRepository.java @@ -27,7 +27,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.util.EntityUtil; /** diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index cd5ae8bb04b3..dac015bb3959 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -57,8 +57,6 @@ import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; import static org.openmetadata.service.exception.CatalogExceptionMessage.notTaskAssignee; -import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; -import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; import static org.openmetadata.service.monitoring.RequestLatencyContext.phase; import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTagsGracefully; @@ -92,26 +90,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.google.common.cache.Weigher; import com.google.common.util.concurrent.UncheckedExecutionException; import com.networknt.schema.Error; import com.networknt.schema.Schema; -import io.micrometer.core.instrument.DistributionSummary; import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.Timer; import jakarta.json.JsonPatch; import jakarta.validation.ConstraintViolationException; -import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.core.Response.Status; import jakarta.ws.rs.core.SecurityContext; import jakarta.ws.rs.core.UriInfo; import java.io.IOException; -import java.io.StringWriter; import java.net.URI; -import java.sql.SQLException; import java.time.Instant; import java.time.LocalDateTime; import java.time.LocalTime; @@ -121,14 +110,12 @@ import java.time.format.DateTimeParseException; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -138,30 +125,18 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.function.BiPredicate; import java.util.function.Consumer; import java.util.function.Function; -import java.util.function.IntSupplier; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; import lombok.Getter; -import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVParser; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -174,7 +149,6 @@ import org.openmetadata.schema.FieldInterface; import org.openmetadata.schema.api.VoteRequest; import org.openmetadata.schema.api.VoteRequest.VoteType; -import org.openmetadata.schema.api.feed.ResolveTask; import org.openmetadata.schema.api.teams.CreateTeam; import org.openmetadata.schema.configuration.AssetCertificationSettings; import org.openmetadata.schema.entity.data.Table; @@ -230,17 +204,14 @@ import org.openmetadata.service.cache.NotFoundCache; import org.openmetadata.service.config.CacheConfiguration; import org.openmetadata.service.events.lifecycle.EntityLifecycleEventDispatcher; -import org.openmetadata.service.exception.BadRequestException; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityLockedException; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.exception.PreconditionFailedException; import org.openmetadata.service.exception.UnhandledServerException; -import org.openmetadata.service.formatter.util.FormatterUtil; -import org.openmetadata.service.governance.workflows.WorkflowHandler; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityVersionPair; -import org.openmetadata.service.jdbi3.CollectionDAO.ExtensionRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityVersionPair; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.ExtensionRecord; import org.openmetadata.service.jdbi3.FeedRepository.TaskWorkflow; import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext; import org.openmetadata.service.jobs.JobDAO; @@ -262,7 +233,6 @@ import org.openmetadata.service.security.policyevaluator.PolicyEvaluator; import org.openmetadata.service.security.policyevaluator.SubjectContext; import org.openmetadata.service.util.EntityETag; -import org.openmetadata.service.util.EntityFieldUtils; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.EntityUtil.RelationIncludes; @@ -274,7 +244,6 @@ import org.openmetadata.service.util.RestUtil.DeleteResponse; import org.openmetadata.service.util.RestUtil.PatchResponse; import org.openmetadata.service.util.RestUtil.PutResponse; -import org.openmetadata.service.workflows.searchIndex.ReindexingUtil; import software.amazon.awssdk.utils.Either; /** @@ -327,122 +296,17 @@ public record EntityHistoryWithOffset(EntityHistory entityHistory, int nextOffse private record InheritanceCacheKey(String entityType, UUID entityId, String fieldsKey) {} - private static final int STRING_OBJECT_OVERHEAD_BYTES = 40; - - // Conservative upper-bound weight for a String: length() * 2 (UTF-16 worst-case) + 40 (header). - // On Java 21 with compact strings, LATIN1 content uses fewer bytes, so this overestimates - // slightly — which is intentional for memory capping. Zero allocation, single field read. - // Defaults used before CacheConfiguration is loaded at startup. initCaches() replaces these. - public static volatile LoadingCache, String> CACHE_WITH_NAME = - buildEntityNameCache( - CacheConfiguration.DEFAULT_ENTITY_CACHE_MAX_SIZE_BYTES, - CacheConfiguration.DEFAULT_ENTITY_CACHE_TTL_SECONDS); - public static volatile LoadingCache, String> CACHE_WITH_ID = - buildEntityIdCache( - CacheConfiguration.DEFAULT_ENTITY_CACHE_MAX_SIZE_BYTES, - CacheConfiguration.DEFAULT_ENTITY_CACHE_TTL_SECONDS); - - /** - * Canonical {@link #CACHE_WITH_NAME} key. User FQNs are lowercased at the DB layer - * ({@code UserDAO.findEntityByName}), so the Guava cache must use the same normalization — - * otherwise {@code Alice@x.com} and {@code alice@x.com} produce two split entries and - * invalidations written against the lowercased canonical form miss the mixed-case entry, - * serving stale data until TTL. - */ - private static Pair cacheNameKey(String entityType, String fqn) { - if (fqn != null && Entity.USER.equals(entityType)) { - return new ImmutablePair<>(entityType, fqn.toLowerCase(Locale.ROOT)); - } - return new ImmutablePair<>(entityType, fqn); - } - - /** - * Rebuild entity caches with values from {@link CacheConfiguration}. Called once during app - * startup after the configuration is loaded. Safe to call multiple times — subsequent calls - * replace the caches (old entries are lost, which is fine during initialization). - */ public static void initCaches(CacheConfiguration config) { - CACHE_WITH_NAME = - buildEntityNameCache( - config.getEntityCacheMaxSizeBytes(), config.getEntityCacheTTLSeconds()); - CACHE_WITH_ID = - buildEntityIdCache(config.getEntityCacheMaxSizeBytes(), config.getEntityCacheTTLSeconds()); - LOG.info( - "Entity caches initialized: maxWeight={}MB, ttl={}s", - config.getEntityCacheMaxSizeBytes() / (1024 * 1024), - config.getEntityCacheTTLSeconds()); - } - - private static LoadingCache, String> buildEntityNameCache( - long maxWeightBytes, int ttlSeconds) { - return CacheBuilder.newBuilder() - .maximumWeight(maxWeightBytes) - .weigher( - (Weigher, String>) - (key, value) -> value.length() * 2 + STRING_OBJECT_OVERHEAD_BYTES) - .expireAfterWrite(ttlSeconds, TimeUnit.SECONDS) - .recordStats() - .build(new EntityLoaderWithName()); - } - - private static LoadingCache, String> buildEntityIdCache( - long maxWeightBytes, int ttlSeconds) { - return CacheBuilder.newBuilder() - .maximumWeight(maxWeightBytes) - .weigher( - (Weigher, String>) - (key, value) -> value.length() * 2 + STRING_OBJECT_OVERHEAD_BYTES) - .expireAfterWrite(ttlSeconds, TimeUnit.SECONDS) - .recordStats() - .build(new EntityLoaderWithId()); - } - - private static final int DEFAULT_FIELD_FETCH_POOL_SIZE = - Math.min(50, Runtime.getRuntime().availableProcessors() * 4); - private static final ThreadPoolExecutor FIELD_FETCH_EXECUTOR = - createFieldFetchExecutor(DEFAULT_FIELD_FETCH_POOL_SIZE); - - private static ThreadPoolExecutor createFieldFetchExecutor(int poolSize) { - ThreadPoolExecutor pool = - new ThreadPoolExecutor( - poolSize, - poolSize, - 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - java.lang.Thread.ofVirtual().name("om-field-fetch-", 0).factory()); - pool.allowCoreThreadTimeOut(true); - return pool; - } - - public static synchronized void setFieldFetchPoolSize(int size) { - int newSize = Math.max(1, Math.min(50, size)); - if (newSize <= FIELD_FETCH_EXECUTOR.getMaximumPoolSize()) { - FIELD_FETCH_EXECUTOR.setCorePoolSize(newSize); - FIELD_FETCH_EXECUTOR.setMaximumPoolSize(newSize); - } else { - FIELD_FETCH_EXECUTOR.setMaximumPoolSize(newSize); - FIELD_FETCH_EXECUTOR.setCorePoolSize(newSize); - } - LOG.info("Field-fetch pool resized to {} threads", newSize); + EntityCaches.initCaches(config); } - public static synchronized void resetFieldFetchPoolSize() { - setFieldFetchPoolSize(DEFAULT_FIELD_FETCH_POOL_SIZE); + public static void setFieldFetchPoolSize(int size) { + EntityCaches.setFieldFetchPoolSize(size); } - private static final LoadingCache COUNT_CACHE = - CacheBuilder.newBuilder() - .maximumSize(500) - .expireAfterWrite(5, TimeUnit.MINUTES) - .recordStats() - .build( - new CacheLoader() { - @Override - public Integer load(String key) { - throw new UnsupportedOperationException("Use get() method with a custom loader"); - } - }); + public static void resetFieldFetchPoolSize() { + EntityCaches.resetFieldFetchPoolSize(); + } private final String collectionPath; @Getter public final Class entityClass; @@ -485,21 +349,18 @@ public Integer load(String key) { protected boolean supportsSearch = false; protected final Map, Fields>> fieldFetchers = new HashMap<>(); + private final BulkFieldFetcher bulkFieldFetcher = new BulkFieldFetcher<>(this); + private final BulkImportService bulkImportService = new BulkImportService<>(this); private final ReadPlanner readPlanner = new ReadPlanner(); /** * Thread-local cache for parent entities during bulk prepare operations. * Set by {@link #preloadParentsForBulk(List)} and cleared after use. */ - private final ThreadLocal> parentCacheForPrepare = new ThreadLocal<>(); - - private static final ThreadLocal> - inheritanceParentCache = ThreadLocal.withInitial(HashMap::new); - protected final ChangeSummarizer changeSummarizer; // Lock manager for preventing orphaned entities during cascade deletion - private static HierarchicalLockManager lockManager; + static HierarchicalLockManager lockManager; // Static setter for lock manager initialization public static void setLockManager(HierarchicalLockManager manager) { @@ -1002,114 +863,36 @@ public void preloadParentsForBulk(List entities) { } /** Get a parent entity from the bulk-prepare cache, or load from DB if not cached. */ + private final InheritanceParentCache inheritanceCache = new InheritanceParentCache(); + protected EntityInterface getCachedParentOrLoad( EntityReference ref, String fields, Include include) { - var cache = parentCacheForPrepare.get(); - if (cache != null && ref != null && ref.getId() != null) { - var cached = cache.get(ref.getId()); - if (cached != null) return cached; - } - return Entity.getEntity(ref, fields, include); + return inheritanceCache.getCachedParentOrLoad(ref, fields, include); } - /** Store preloaded parents in the thread-local cache. */ protected void setParentCache(Map cache) { - parentCacheForPrepare.set(cache); + inheritanceCache.setParentCache(cache); } - /** Clear the parent cache after bulk prepare. */ public void clearParentCache() { - parentCacheForPrepare.remove(); - inheritanceParentCache.remove(); + inheritanceCache.clearParentCache(); } public static void clearInheritanceParentCache() { - inheritanceParentCache.remove(); + InheritanceParentCache.clearInheritanceParentCache(); } - private EntityInterface getCachedInheritanceParent(EntityReference parentRef, String fields) { - if (parentRef == null || parentRef.getId() == null || nullOrEmpty(parentRef.getType())) { - return null; - } - Map cache = inheritanceParentCache.get(); - InheritanceCacheKey directKey = inheritanceCacheKey(parentRef, fields); - EntityInterface direct = cache.get(directKey); - if (direct != null) { - return direct; - } - - // Reuse a superset entry when the same parent was already loaded with broader fields - // (for example "owners,domains,retentionPeriod" can serve "owners,domains"). - Set requestedFields = parseFieldSet(directKey.fieldsKey()); - for (Entry entry : cache.entrySet()) { - InheritanceCacheKey cachedKey = entry.getKey(); - if (!cachedKey.entityType().equals(parentRef.getType()) - || !cachedKey.entityId().equals(parentRef.getId())) { - continue; - } - if (parseFieldSet(cachedKey.fieldsKey()).containsAll(requestedFields)) { - return entry.getValue(); - } - } - return null; + EntityInterface getCachedInheritanceParent(EntityReference parentRef, String fields) { + return inheritanceCache.getCachedInheritanceParent(parentRef, fields); } protected final

P getOrLoadInheritanceParent( EntityReference parentRef, String fields, Class

parentClass) { - if (parentRef == null || parentRef.getId() == null || nullOrEmpty(parentRef.getType())) { - return null; - } - EntityInterface parent = getCachedInheritanceParent(parentRef, fields); - if (parent == null) { - parent = Entity.getEntityForInheritance(parentRef.getType(), parentRef.getId(), fields, ALL); - cacheInheritanceParent(parentRef, fields, parent); - } - if (!parentClass.isInstance(parent)) { - return null; - } - return parentClass.cast(parent); - } - - private void cacheInheritanceParent( - EntityReference parentRef, String fields, EntityInterface parent) { - if (parentRef == null || parentRef.getId() == null || nullOrEmpty(parentRef.getType())) { - return; - } - if (parent == null || parent.getId() == null) { - return; - } - inheritanceParentCache.get().put(inheritanceCacheKey(parentRef, fields), parent); - } - - private InheritanceCacheKey inheritanceCacheKey(EntityReference parentRef, String fields) { - return new InheritanceCacheKey( - parentRef.getType(), parentRef.getId(), normalizeFieldList(fields)); - } - - private String normalizeFieldList(String fields) { - if (fields == null || fields.isBlank()) { - return ""; - } - // Canonicalize field order so cache keys are stable across equivalent requests - // (e.g. "owners,domains" and "domains, owners" should share the same parent entry). - return fields - .lines() - .flatMap(line -> Stream.of(line.split(","))) - .map(String::trim) - .filter(field -> !field.isEmpty()) - .distinct() - .sorted() - .collect(Collectors.joining(",")); + return inheritanceCache.getOrLoadInheritanceParent(parentRef, fields, parentClass); } - private Set parseFieldSet(String fields) { - if (fields == null || fields.isBlank()) { - return Collections.emptySet(); - } - return Stream.of(fields.split(",")) - .map(String::trim) - .filter(field -> !field.isEmpty()) - .collect(Collectors.toSet()); + void cacheInheritanceParent(EntityReference parentRef, String fields, EntityInterface parent) { + inheritanceCache.cacheInheritanceParent(parentRef, fields, parent); } /** @@ -1378,7 +1161,7 @@ public final T get( } if (!fromCache) { - CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); } T entity = withPhase("entityLookup", () -> find(id, relationIncludes.getDefaultInclude(), fromCache)); @@ -1467,7 +1250,7 @@ public final T find(UUID id, Include include, boolean fromCache) throws EntityNo && notFoundCache.isMarkedNotFoundById(entityType, id)) { throw new EntityNotFoundException(entityNotFound(entityType, id)); } - CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); T entity; try (var ignored = phase("dbFindByIdNoCache")) { entity = dao.findEntityById(id, include); @@ -1499,7 +1282,7 @@ public final T find(UUID id, Include include, boolean fromCache) throws EntityNo // — check NotFoundCache unconditionally — was a hot-path regression for every L1 hit. try { ImmutablePair cacheKey = new ImmutablePair<>(entityType, id); - String cachedJson = CACHE_WITH_ID.getIfPresent(cacheKey); + String cachedJson = EntityCaches.CACHE_WITH_ID.getIfPresent(cacheKey); if (cachedJson == null) { // L1 miss. Consult the negative cache so we can short-circuit before invoking the // loader (which would do DB + optional Redis-L2 work). @@ -1509,7 +1292,7 @@ public final T find(UUID id, Include include, boolean fromCache) throws EntityNo throw new EntityNotFoundException(entityNotFound(entityType, id)); } try (var ignored = phase("cacheGet")) { - cachedJson = CACHE_WITH_ID.get(cacheKey); + cachedJson = EntityCaches.CACHE_WITH_ID.get(cacheKey); } } T entity; @@ -1521,7 +1304,7 @@ public final T find(UUID id, Include include, boolean fromCache) throws EntityNo if (entity != null && entity.getId() == null) { LOG.error( "CRITICAL: Entity from cache has null ID! Type: {}, Expected ID: {}", entityType, id); - CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); entity = dao.findEntityById(id, include); if (entity == null) { throw new EntityNotFoundException(entityNotFound(entityType, id)); @@ -1584,7 +1367,7 @@ public final T getByName( } if (!fromCache) { - CACHE_WITH_NAME.invalidate(cacheNameKey(entityType, fqn)); + EntityCaches.CACHE_WITH_NAME.invalidate(EntityCaches.cacheNameKey(entityType, fqn)); } T entity; try (var ignored = phase("entityLookup")) { @@ -2099,7 +1882,7 @@ public final T findByName(String fqn, Include include, boolean fromCache) { && notFoundCache.isMarkedNotFoundByName(entityType, fqn)) { throw new EntityNotFoundException(entityNotFound(entityType, fqn)); } - CACHE_WITH_NAME.invalidate(cacheNameKey(entityType, fqn)); + EntityCaches.CACHE_WITH_NAME.invalidate(EntityCaches.cacheNameKey(entityType, fqn)); T entity; try (var ignored = phase("dbFindByNameNoCache")) { entity = dao.findEntityByName(fqn, include); @@ -2119,8 +1902,8 @@ public final T findByName(String fqn, Include include, boolean fromCache) { // Hot path — L1 Guava first, NotFoundCache only on L1 miss. Same shape as find(UUID,…). try { - Pair cacheKey = cacheNameKey(entityType, fqn); - String cachedJson = CACHE_WITH_NAME.getIfPresent(cacheKey); + Pair cacheKey = EntityCaches.cacheNameKey(entityType, fqn); + String cachedJson = EntityCaches.CACHE_WITH_NAME.getIfPresent(cacheKey); if (cachedJson == null) { if (include == NON_DELETED && notFoundCache != null @@ -2128,7 +1911,7 @@ public final T findByName(String fqn, Include include, boolean fromCache) { throw new EntityNotFoundException(entityNotFound(entityType, fqn)); } try (var ignored = phase("cacheGet")) { - cachedJson = CACHE_WITH_NAME.get(cacheKey); + cachedJson = EntityCaches.CACHE_WITH_NAME.get(cacheKey); } } T entity; @@ -2503,7 +2286,7 @@ public final EntityHistory listVersions(UUID id) { private int getVersionCountCached(String tableName, long startTs, long endTs, String entityType) { String cacheKey = String.format("%s:%d:%d:%s", tableName, startTs, endTs, entityType); try { - return COUNT_CACHE.get( + return EntityCaches.COUNT_CACHE.get( cacheKey, () -> daoCollection @@ -2934,445 +2717,15 @@ public final T setFieldsInternal(T entity, Fields fields, RelationIncludes relat return entity; } - /** - * Invalidate cache entries for every descendant of {@code oldPrefix} in the given entity type's - * DB table. Called by rename-cascade flows (e.g. DomainRepository.updateName) right before the - * bulk {@code UPDATE ... WHERE fqnHash LIKE 'oldPrefix.%'} so downstream reads don't see the - * stale (pre-rename) FQN on the children. - * - *

Publishes pub/sub for each descendant so peer OM instances drop their Guava entries too. - * - *

Returns the enumerated {@code (id, oldFqn)} pairs so the caller can pass them to {@link - * #finishInvalidateCacheForRenameCascade} once the rename-related DB statements have run — - * necessary because a reader landing in the window between this call and the bulk - * {@code UPDATE} can repopulate the by-id cache with the still-visible pre-rename row, and - * only a second invalidate pass after the DB statement can evict the poisoned entry. - * - *

Transactional scope: the existing rename call sites invoke both passes inside the - * same {@code @Transaction}-annotated updater, so the {@code finish} pass runs after the bulk - * {@code UPDATE} statement(s) but before the surrounding transaction commits. That - * closes the wide pre-update window (seconds, dominated by search-index walks) that CI - * traced as the failure mode, but a residual race remains: a concurrent reader landing - * between the {@code finish} pass and commit can still see the pre-rename row under - * READ COMMITTED and repopulate the cache. The window is on the order of milliseconds and - * we have no integration failures attributed to it; a true after-commit hook would close it - * fully and is tracked as a follow-up. - * - * @param entityType type name (e.g. {@code domain}, {@code dataProduct}, {@code tag}) - * @param oldPrefix fully qualified name prefix the rename is moving away from - */ - public static List invalidateCacheForRenameCascade( - String entityType, String oldPrefix) { - if (entityType == null || nullOrEmpty(oldPrefix)) { - return Collections.emptyList(); - } - EntityRepository repo; - try { - repo = Entity.getEntityRepository(entityType); - } catch (Exception e) { - return Collections.emptyList(); - } - if (repo == null || repo.getDao() == null) { - return Collections.emptyList(); - } - List affected; - try { - affected = repo.getDao().listDescendantIdFqnByPrefix(oldPrefix); - } catch (Exception e) { - LOG.warn( - "Failed to enumerate descendants for cache invalidation: type={} prefix={}", - entityType, - oldPrefix, - e); - return Collections.emptyList(); - } - if (affected.isEmpty()) { - return Collections.emptyList(); - } - dropDescendantCacheEntries(entityType, affected); - LOG.info( - "Invalidated cache for {} descendants of rename cascade: type={} prefix={}", - affected.size(), - entityType, - oldPrefix); - return affected; - } - - /** - * Post-rename-write pair to {@link #invalidateCacheForRenameCascade}. Re-evicts the cached - * forms of every descendant captured before the rename — by id and by the old FQN. Closes - * the wide race window where a concurrent reader arriving in the seconds between the - * pre-invalidate and the bulk rename {@code UPDATE} repopulates the by-id (or by-old-fqn) - * cache with the still-visible pre-rename row and pins that staleness for the entity TTL. - * - *

Called inside the same transaction as the rename writes (see {@link - * #invalidateCacheForRenameCascade} for the full transactional caveat); the millisecond - * window between this pass and commit is still racy but is not the failure mode CI traced. - * - *

Safe to call with an empty or null list (no-op). - */ - public static void finishInvalidateCacheForRenameCascade( - String entityType, List affected) { - if (entityType == null || affected == null || affected.isEmpty()) { - return; - } - dropDescendantCacheEntries(entityType, affected); - LOG.debug( - "Post-rename-write re-invalidated cache for {} descendants: type={}", - affected.size(), - entityType); - } - - private static void dropDescendantCacheEntries( - String entityType, List affected) { - // Rename cascades run inside the rename flush transaction. Route each descendant through - // invalidateCacheForEntity so the Guava-L1 eviction stays inline (cheap) while the Redis-L2 - // round trip is deferred to the post-commit drain when a flush scope is open — never issued - // while the pooled connection is held. The previous per-row pub/sub reason label was - // informational only (remote listeners evict L1 regardless), so the unified path is equivalent. - for (EntityDAO.EntityIdFqnPair row : affected) { - invalidateCacheForEntity(entityType, row.id, row.fqn); - } - } - - /** - * When a cache-deferral scope is open on the calling thread, {@link #invalidateCacheForEntity} - * records the Redis-L2 invalidation key here instead of issuing the (blocking, syncCommands) - * Redis round trip inline. The flush opens a scope before its DB transaction so no Redis call - * runs while a pooled connection is held — only the cheap local Guava-L1 invalidate stays inline - * — then drains the de-duplicated keys after commit, on the request thread, preserving - * read-your-write. {@code null} means "no scope active" and the Redis-L2 work runs inline. - */ - private static final ThreadLocal> - DEFERRED_CACHE_INVALIDATIONS = new ThreadLocal<>(); - - /** - * De-duplication key for a deferred Redis-L2 cache invalidation. Equality is on {@code - * (entityType, id)} only so repeated relationship writes touching the same entity collapse to one - * post-commit invalidation; the {@code fqn} is carried along (best non-null wins) so the by-name - * Redis variant is evicted when any caller knew the FQN. - */ - private static final class CacheInvalidationKey { - private final String entityType; - private final UUID id; - private final String fqn; - - private CacheInvalidationKey(String entityType, UUID id, String fqn) { - this.entityType = entityType; - this.id = id; - this.fqn = fqn; - } - - @Override - public boolean equals(Object other) { - boolean result = this == other; - if (!result && other instanceof CacheInvalidationKey key) { - result = entityType.equals(key.entityType) && id.equals(key.id); - } - return result; - } - - @Override - public int hashCode() { - return Objects.hash(entityType, id); - } - } - - /** - * Open a Redis-L2 cache-invalidation deferral scope on the current thread. While open, {@link - * #invalidateCacheForEntity} records keys instead of issuing the blocking Redis round trip. - * Returns {@code true} if this call opened the scope (caller owns draining/closing it), {@code - * false} if a scope was already open (nested call — the outer owner stays responsible). Pair a - * {@code true} result with {@link #drainCacheInvalidations()} after commit and {@link - * #clearCacheInvalidations()} on failure. - */ - static boolean beginCacheInvalidationDeferral() { - boolean opened = DEFERRED_CACHE_INVALIDATIONS.get() == null; - if (opened) { - DEFERRED_CACHE_INVALIDATIONS.set(new LinkedHashMap<>()); - } - return opened; - } - - /** Run every de-duplicated Redis-L2 invalidation captured since the scope opened, then close it. */ - static void drainCacheInvalidations() { - Map deferred = DEFERRED_CACHE_INVALIDATIONS.get(); - DEFERRED_CACHE_INVALIDATIONS.remove(); - if (deferred != null) { - for (CacheInvalidationKey key : deferred.values()) { - invalidateRedisL2ForEntity(key.entityType, key.id, key.fqn); - } - } - } - - /** Discard captured keys and close the scope without running them (failed transaction). */ - static void clearCacheInvalidations() { - DEFERRED_CACHE_INVALIDATIONS.remove(); - } - - /** - * Full local + cross-instance cache eviction for a single entity. Used by code paths that - * update a referring entity indirectly (e.g. data-product domain change updates the linked - * tables; a tag delete affects policies that embed it). Does the same work as - * {@link #invalidateCache(EntityInterface)} but doesn't require the full entity POJO — the - * {@code (type, id, fqn)} triple is enough to drop every cached variant. - * - *

The Guava-L1 eviction always runs inline (local, cheap). The Redis-L2 portion (base/by-name - * hash, relationship, bundle, lineage, pub/sub) issues a blocking {@code syncCommands} round - * trip, so when a deferral scope is active (a flush is holding a pooled DB connection) it is - * recorded and replayed post-commit on the request thread instead — never inside the handle. - */ - public static void invalidateCacheForEntity(String entityType, UUID id, String fqn) { - if (entityType == null || id == null) { - return; - } - // Guava L1 always cleared inline — it is a local map eviction, not a network round trip, and - // the rare uncached read path may have populated it even for UNCACHED_ENTITY_TYPES. - CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); - if (fqn != null) { - CACHE_WITH_NAME.invalidate(cacheNameKey(entityType, fqn)); - } - // Skip every Redis op for entity types that are never cached. Bot/domain/data-product - // deletes cascade through many addRelationship/deleteRelationship calls; without this - // short-circuit each cascade pays for a pub/sub publish + multiple DELs that touch keys - // we never wrote — under heavy parallel load that pushes test budgets like - // TaskResourceIT.testDeletingBotCreatorCleansUpOpenSuggestionTasks past their 30 s window. - if (!isCacheableEntityType(entityType)) { - return; - } - Map deferred = DEFERRED_CACHE_INVALIDATIONS.get(); - if (deferred != null) { - recordCacheInvalidation(deferred, entityType, id, fqn); - } else { - invalidateRedisL2ForEntity(entityType, id, fqn); - } - } - - private static void recordCacheInvalidation( - Map deferred, - String entityType, - UUID id, - String fqn) { - CacheInvalidationKey key = new CacheInvalidationKey(entityType, id, fqn); - CacheInvalidationKey existing = deferred.putIfAbsent(key, key); - if (existing != null && existing.fqn == null && fqn != null) { - deferred.put(key, key); - } - } - - private static void invalidateRedisL2ForEntity(String entityType, UUID id, String fqn) { - var cachedEntityDao = CacheBundle.getCachedEntityDao(); - if (cachedEntityDao != null) { - cachedEntityDao.invalidateBase(entityType, id); - if (fqn != null) { - cachedEntityDao.invalidateByName(entityType, fqn); - } - } - var cachedRelationshipDao = CacheBundle.getCachedRelationshipDao(); - if (cachedRelationshipDao != null) { - cachedRelationshipDao.invalidateOwners(entityType, id); - cachedRelationshipDao.invalidateDomains(entityType, id); - cachedRelationshipDao.invalidateContainer(entityType, id); - } - var cachedReadBundle = CacheBundle.getCachedReadBundle(); - if (cachedReadBundle != null) { - cachedReadBundle.invalidate(entityType, id); - } - var cachedLineage = CacheBundle.getCachedLineage(); - if (cachedLineage != null) { - cachedLineage.invalidate(id); - } - var pubsub = CacheBundle.getCacheInvalidationPubSub(); - if (pubsub != null) { - pubsub.publish(entityType, id, fqn, "ref-change"); - } - } - - /** - * Invalidate cache entries for an entity identified by an {@link - * CollectionDAO.EntityRelationshipRecord}. Extracts {@code fullyQualifiedName} from the record's - * JSON payload (when present) so the by-name cache variant is evicted alongside the by-id one. - * Callers that only have {@code (type, id)} and pass {@code fqn=null} leave GET-by-name entries - * stale until TTL expiry — use this when the referenced entity's FQN needs to be invalidated too. - */ - public static void invalidateCacheForReferencedEntity( - CollectionDAO.EntityRelationshipRecord record) { - if (record == null) { - return; - } - invalidateCacheForEntity(record.getType(), record.getId(), extractFqn(record.getJson())); - } - - /** - * Drop cached entity JSON, bundle, and relationship caches for every entity that carries the - * given tag FQN. The {@code tag_usage} table only stores {@code targetFQNHash}, so we cannot - * cheaply derive (type, id, fqn) from it; we lean on the search index instead — the same source - * the search-side {@code updateClassificationTagByFqnPrefix} reindex uses to find affected - * documents. Run this BEFORE the search reindex runs so the search query still matches documents - * by the old tag FQN. - * - *

Consistency tradeoff: coverage is bounded by search-index freshness. Entities - * tagged recently enough that the indexer hasn't picked them up are missed and fall back to - * the entity TTL (default 48h). On busy clusters with replication lag this can be minutes. - * If strict consistency is ever required, a direct {@code tag_usage} table query joined back - * to each candidate entity table would be more reliable at the cost of one round-trip per - * candidate type. - */ - public static int invalidateCacheForTaggedEntities(String tagFqn) { - int result = 0; - if (!nullOrEmpty(tagFqn)) { - result = - deferOrRunSearchBackedInvalidation( - () -> searchTaggedEntitiesAndInvalidate(tagFqn), tagFqn); - } - return result; - } - - /** - * Run the search-backed cache invalidation for {@code tagFqn} inline when no flush deferral scope - * is open, or capture it for post-commit drain when a rename/move cascade has opened one — the - * blocking ES search loop must never run while the DB transaction handle is held, and it must run - * exactly once even when a deadlock replays the cascade. The inline (non-flush) result is the live - * invalidated count; the deferred path returns {@code 0} because the work runs after this method - * returns and the count is only known then (it is logged inside {@code - * searchTaggedEntitiesAndInvalidate}). - */ - private static int deferOrRunSearchBackedInvalidation(IntSupplier invalidation, String tagFqn) { - int result = 0; - if (SearchRepository.isSearchWriteDeferralActive()) { - SearchRepository.deferOrRunSearchWrite( - invalidation::getAsInt, "invalidateCacheForTaggedEntities", null, tagFqn, null); - } else { - result = invalidation.getAsInt(); - } - return result; - } - - private static int searchTaggedEntitiesAndInvalidate(String tagFqn) { - int total = 0; - int from = 0; - boolean exhausted = false; - while (!exhausted) { - List page = findTaggedEntitiesPage(tagFqn, from); - if (page.isEmpty()) { - exhausted = true; - } else { - for (EntityReference ref : page) { - invalidateCacheForEntity(ref.getType(), ref.getId(), ref.getFullyQualifiedName()); - total++; - } - from += page.size(); - } - } - if (total > 0) { - LOG.info("Invalidated cache for {} entities tagged with: {}", total, tagFqn); - } - return total; - } - - private static List findTaggedEntitiesPage(String tagFqn, int from) { - List page; - try { - page = - ReindexingUtil.findReferenceInElasticSearchAcrossAllIndexes( - "tags.tagFQN", ReindexingUtil.escapeDoubleQuotes(tagFqn), from); - } catch (Exception e) { - LOG.warn("Search-based cache invalidation failed for tag={}", tagFqn, e); - page = List.of(); - } - return page; - } - - /** Bulk variant — invalidates entities tagged with any of the supplied tag FQNs. */ - public static int invalidateCacheForTaggedEntities(Collection tagFqns) { - int total = 0; - if (!nullOrEmpty(tagFqns)) { - for (String fqn : tagFqns) { - total += invalidateCacheForTaggedEntities(fqn); - } - if (total > 0) { - LOG.info( - "Invalidated cache for {} entities across {} renamed tag FQNs", total, tagFqns.size()); - } - } - return total; - } - - /** - * Convenience wrapper for tag-like entity renames (Tag, GlossaryTerm) where the rename cascades - * to descendants in the same entity table. Enumerates the descendant FQNs from the entity DAO - * BEFORE the DB rename rewrites them, then invalidates cached entities tagged with the prefix or - * any descendant. For Classification (where children live in a different entity table), enumerate - * child tag FQNs at the call site and pass them to {@link - * #invalidateCacheForTaggedEntities(Collection)} directly. - */ - public static int invalidateCacheForTaggedEntitiesAndDescendants( - String entityType, String oldPrefix) { - if (entityType == null || nullOrEmpty(oldPrefix)) { - return 0; - } - List fqns = new ArrayList<>(); - fqns.add(oldPrefix); - try { - EntityRepository repo = Entity.getEntityRepository(entityType); - if (repo != null && repo.getDao() != null) { - List descendants = - repo.getDao().listDescendantIdFqnByPrefix(oldPrefix); - for (EntityDAO.EntityIdFqnPair pair : descendants) { - if (pair.fqn != null && !pair.fqn.equals(oldPrefix)) { - fqns.add(pair.fqn); - } - } - } - } catch (Exception e) { - LOG.warn( - "Failed to enumerate descendants for tagged-entity invalidation: type={} fqn={}", - entityType, - oldPrefix, - e); - } - return invalidateCacheForTaggedEntities(fqns); - } - - private static String extractFqn(String json) { - if (json == null || json.isEmpty()) { - return null; - } - try { - var node = JsonUtils.readTree(json); - return node.hasNonNull("fullyQualifiedName") ? node.get("fullyQualifiedName").asText() : null; - } catch (Exception e) { - LOG.debug("Failed to extract fullyQualifiedName for cache invalidation", e); - return null; - } - } - - /** - * Invoked by {@link org.openmetadata.service.cache.CacheInvalidationPubSub} when another OM - * instance signals an entity change. Evicts this instance's per-process Guava caches so the next - * read pulls fresh data. Does not touch Redis — the writer already invalidated shared keys - * before publishing. - */ - public static void onRemoteCacheInvalidate(String entityType, UUID id, String fqn) { - if (entityType == null) { - return; - } - if (id != null) { - CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); - } - if (fqn != null) { - CACHE_WITH_NAME.invalidate(cacheNameKey(entityType, fqn)); - } - } - /** * Invalidate cache entries when entity is deleted */ protected void invalidateCache(T entity) { try { // Invalidate Guava LoadingCache entries - CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, entity.getId())); - CACHE_WITH_NAME.invalidate(cacheNameKey(entityType, entity.getFullyQualifiedName())); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, entity.getId())); + EntityCaches.CACHE_WITH_NAME.invalidate( + EntityCaches.cacheNameKey(entityType, entity.getFullyQualifiedName())); // Invalidate Redis cache entries var cachedEntityDao = CacheBundle.getCachedEntityDao(); @@ -3626,7 +2979,8 @@ public List updateManyEntitiesForImport( // freshly-stored row + relationships. writeThroughCacheMany only populates Redis base // entries; Guava and bundle caches still serve pre-update tags/owners/etc. until TTL. for (T entity : updatedEntities) { - invalidateCacheForEntity(entityType, entity.getId(), entity.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + entityType, entity.getId(), entity.getFullyQualifiedName()); } }); @@ -3717,7 +3071,7 @@ static boolean isCacheableEntityType(String entityType) { /** * Validates entity has required fields for caching */ - private static boolean isValidEntityForCache(EntityInterface entity) { + static boolean isValidEntityForCache(EntityInterface entity) { return entity != null && entity.getId() != null && entity.getFullyQualifiedName() != null; } @@ -4154,7 +3508,8 @@ public void patchChangeSummary( dao.update(entity.getId(), entity.getFullyQualifiedName(), JsonUtils.pojoToJson(entity)); // Direct dao.update skips invalidateCachesAfterStore, so drop every cached variant so the // next read picks up the new changeSummary instead of serving stale JSON. - invalidateCacheForEntity(entityType, entity.getId(), entity.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + entityType, entity.getId(), entity.getFullyQualifiedName()); } @Transaction @@ -4609,8 +3964,9 @@ protected void entitySpecificCleanup(String deletedBy, T entityInterface) { } void invalidate(T entity) { - CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, entity.getId())); - CACHE_WITH_NAME.invalidate(cacheNameKey(entityType, entity.getFullyQualifiedName())); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, entity.getId())); + EntityCaches.CACHE_WITH_NAME.invalidate( + EntityCaches.cacheNameKey(entityType, entity.getFullyQualifiedName())); RequestEntityCache.invalidate(entityType, entity.getId(), entity.getFullyQualifiedName()); // Also invalidate Redis cache @@ -4679,7 +4035,7 @@ public final ResultList getResultList( * connection and starve the pool. Tag RDF is deferred via {@link RdfTagUpdater#beginDeferral()}, * the domain/data-product lineage-ES leaf via {@link LineageUtil#beginLineageDeferral()}, and the * Redis-L2 cache invalidation issued by {@code addRelationship}/{@code deleteRelationship}/{@code - * invalidateCacheForEntity} via {@link #beginCacheInvalidationDeferral()} — all drained + * invalidateCacheForEntity} via {@link #EntityCacheInvalidator.beginCacheInvalidationDeferral()} — all drained * post-commit on the request thread. Only the cheap local Guava-L1 eviction stays inline. Redis * cache write-through likewise happens post-commit on the request thread (read-your-write safe). */ @@ -4785,7 +4141,7 @@ private void openCollectors() { ownsRdf = RdfTagUpdater.beginDeferral(); ownsLineageEs = LineageUtil.beginLineageDeferral(); ownsSearchWrite = SearchRepository.beginSearchWriteDeferral(); - ownsCache = beginCacheInvalidationDeferral(); + ownsCache = EntityCacheInvalidator.beginCacheInvalidationDeferral(); } /** @@ -4815,8 +4171,8 @@ private void resetForReplay() { SearchRepository.rollbackSearchWriteToCheckpoint(searchWriteCheckpoint); } if (ownsCache) { - clearCacheInvalidations(); - beginCacheInvalidationDeferral(); + EntityCacheInvalidator.clearCacheInvalidations(); + EntityCacheInvalidator.beginCacheInvalidationDeferral(); } } @@ -4834,7 +4190,7 @@ private void finish(boolean committed) { * rename-cascade search rewrites. Running inline post-commit keeps search and lineage * read-your-write visible by the time the request returns. Every collector's thread-local is * removed UP FRONT and each run step is guarded, so one failing step (e.g. a Redis round trip in - * {@code drainCacheInvalidations}) can never strand a later collector's thread-local on a reused + * {@code EntityCacheInvalidator.drainCacheInvalidations}) can never strand a later collector's thread-local on a reused * request thread, which would otherwise silently drop the next request's deferred writes. */ private void drain() { @@ -4844,7 +4200,7 @@ private void drain() { List searchClosures = ownsSearchWrite ? SearchRepository.drainSearchWriteDeferred() : List.of(); if (ownsCache) { - runGuarded(EntityRepository::drainCacheInvalidations); + runGuarded(EntityCacheInvalidator::drainCacheInvalidations); } runGuarded(() -> RdfTagUpdater.runDeferredClosures(rdfClosures)); runGuarded(() -> runLineageEsClosures(lineageClosures)); @@ -4862,7 +4218,7 @@ private void clear() { SearchRepository.clearSearchWriteDeferred(); } if (ownsCache) { - clearCacheInvalidations(); + EntityCacheInvalidator.clearCacheInvalidations(); } } } @@ -4938,7 +4294,7 @@ private void enqueueLineageEsRetry(EntityReference toEntity, Exception failure) } } - private List createManyEntities(List entities) { + List createManyEntities(List entities) { createManyEntitiesFlush(entities); try (var ignored = phase("postCreate")) { postCreate(entities); @@ -6703,8 +6059,8 @@ public final void addRelationship( // Drop cached bundle/owners/domains/container for both sides — relationship just changed and // any cached EntityReference list on either side is now stale. The FQN is unknown here so // by-name eviction is skipped; by-id and bundle eviction is what callers actually need. - invalidateCacheForEntity(fromEntity, fromId, null); - invalidateCacheForEntity(toEntity, toId, null); + EntityCacheInvalidator.invalidateCacheForEntity(fromEntity, fromId, null); + EntityCacheInvalidator.invalidateCacheForEntity(toEntity, toId, null); } @Transaction @@ -6727,12 +6083,12 @@ public final void bulkRemoveToRelationship( private static void invalidateForBulkRelationship( UUID fromId, List toIds, String fromEntity, String toEntity) { - invalidateCacheForEntity(fromEntity, fromId, null); + EntityCacheInvalidator.invalidateCacheForEntity(fromEntity, fromId, null); if (toIds == null) { return; } for (UUID toId : toIds) { - invalidateCacheForEntity(toEntity, toId, null); + EntityCacheInvalidator.invalidateCacheForEntity(toEntity, toId, null); } } @@ -7035,8 +6391,8 @@ public final void deleteRelationship( .withRelationshipType(relationship); RdfUpdater.removeRelationship(entityRelationship); // Drop cached bundle/owners/domains/container on both sides — same reason as addRelationship. - invalidateCacheForEntity(fromEntityType, fromId, null); - invalidateCacheForEntity(toEntityType, toId, null); + EntityCacheInvalidator.invalidateCacheForEntity(fromEntityType, fromId, null); + EntityCacheInvalidator.invalidateCacheForEntity(toEntityType, toId, null); } public final void deleteTo( @@ -7563,7 +6919,8 @@ protected BulkOperationResult bulkAssetsOperation( // still show the old domain. Drop the asset's cache so the next read reloads from DB // and re-derives the inherited view. Same is true for the by-name cache and the // shared per-pod Guava caches; invalidateCacheForEntity does all of them. - invalidateCacheForEntity(ref.getType(), ref.getId(), ref.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + ref.getType(), ref.getId(), ref.getFullyQualifiedName()); success.add(new BulkResponse().withRequest(ref)); result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); @@ -9838,12 +9195,12 @@ private void invalidateCachesAfterStore() { String fqn = updated.getFullyQualifiedName(); // Evict the Guava L1 so future reads reload from Redis/DB. - CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); - CACHE_WITH_NAME.invalidate(cacheNameKey(entityType, fqn)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, id)); + EntityCaches.CACHE_WITH_NAME.invalidate(EntityCaches.cacheNameKey(entityType, fqn)); // A rename leaves the old FQN pointing at the now-stale entity; drop that key too so // getByName(oldFqn) misses and falls through to a 404 from DB. if (originalFqn != null && !originalFqn.equals(fqn)) { - CACHE_WITH_NAME.invalidate(cacheNameKey(entityType, originalFqn)); + EntityCaches.CACHE_WITH_NAME.invalidate(EntityCaches.cacheNameKey(entityType, originalFqn)); } // Critical: drop Redis *base* entries for the entity BEFORE writeThroughCache repopulates. @@ -10215,250 +9572,14 @@ private void updateColumnScale(String fieldPrefix, Column origColumn, Column upd } } - static class EntityLoaderWithName extends CacheLoader, String> { - @Override - public @NonNull String load(@NotNull Pair fqnPair) { - String entityType = fqnPair.getLeft(); - String fqn = fqnPair.getRight(); - EntityRepository repository = - Entity.getEntityRepository(entityType); - EntityDAO dao = repository.getDao(); - - // Try to load from external cache first (read-through) for cacheable entity types. - if (isCacheableEntityType(entityType)) { - var cachedEntityDao = CacheBundle.getCachedEntityDao(); - if (cachedEntityDao != null) { - Optional cachedJson = cachedEntityDao.getByName(entityType, fqn); - if (cachedJson.isPresent()) { - LOG.debug("CACHE HIT: Loading entity by name from Redis cache: {} {}", entityType, fqn); - try { - Class entityClass = repository.getEntityClass(); - EntityInterface entity = JsonUtils.readValue(cachedJson.get(), entityClass); - if (entity.getId() == null || entity.getFullyQualifiedName() == null) { - LOG.error( - "CACHE ERROR: Cached entity from name lookup is invalid! Evicting. Type: {}, Name: {}", - entityType, - fqn); - cachedEntityDao.deleteByName(entityType, fqn); - } else { - return cachedJson.get(); - } - } catch (Exception e) { - LOG.warn( - "Failed to deserialize cached entity, evicting and falling back to database: {} {}", - entityType, - fqn, - e); - try { - cachedEntityDao.deleteByName(entityType, fqn); - } catch (Exception evictError) { - LOG.debug( - "Failed to evict bad cache entry by name: {} {}", entityType, fqn, evictError); - } - } - } - LOG.debug("CACHE MISS: Entity not in Redis cache by name: {} {}", entityType, fqn); - } - } - - // Load raw JSON from database. User entities store nameHash off the lowercased FQN — - // UserDAO.findEntityByName lowercases the input. We call dao.findByName directly here - // to stay in the JSON-only path, so mirror the same case-fold for user types. - String lookupFqn = "user".equals(entityType) ? fqn.toLowerCase() : fqn; - LOG.debug("Loading entity by name from database: {} {}", entityType, lookupFqn); - String json = - dao.findByName( - dao.getTableName(), dao.getNameHashColumn(), lookupFqn, dao.getCondition(ALL)); - if (json == null) { - throw new EntityNotFoundException( - String.format("Entity not found: %s %s", entityType, fqn)); - } - - // Validate - EntityInterface entity = JsonUtils.readValue(json, repository.getEntityClass()); - if (!isValidEntityForCache(entity)) { - LOG.error( - "CRITICAL: Entity loaded from database by name is invalid! Type: {}, Name: {}, ID: {}", - entityType, - fqn, - entity == null ? "null" : entity.getId()); - throw new IllegalStateException( - String.format("Invalid entity from database: %s %s", entityType, fqn)); - } - - // Populate Redis on miss so subsequent reads (incl. cross-instance) can hit cache - if (isCacheableEntityType(entityType)) { - var cachedEntityDao = CacheBundle.getCachedEntityDao(); - if (cachedEntityDao != null) { - try { - cachedEntityDao.putByName(entityType, fqn, json); - if (entity.getId() != null) { - cachedEntityDao.putBase(entityType, entity.getId(), json); - } - } catch (Exception e) { - LOG.debug("Failed to populate Redis on byName miss: {} {}", entityType, fqn, e); - } - } - } - - return json; - } - } - - static class EntityLoaderWithId extends CacheLoader, String> { - @Override - public @NonNull String load(@NotNull Pair idPair) { - String entityType = idPair.getLeft(); - UUID id = idPair.getRight(); - EntityRepository repository = - Entity.getEntityRepository(entityType); - EntityDAO dao = repository.getDao(); - - // Try to load from external cache first (read-through) for cacheable entity types. - if (isCacheableEntityType(entityType)) { - var cachedEntityDao = CacheBundle.getCachedEntityDao(); - if (cachedEntityDao != null) { - String cachedJson = cachedEntityDao.getBase(id, entityType); - if (cachedJson != null && !cachedJson.isEmpty()) { - LOG.debug("CACHE HIT: Loading entity from Redis cache: {} {}", entityType, id); - try { - Class entityClass = repository.getEntityClass(); - EntityInterface entity = JsonUtils.readValue(cachedJson, entityClass); - if (entity.getId() == null) { - LOG.error( - "CACHE ERROR: Cached entity has null ID! Evicting. Type: {}, Expected ID: {}", - entityType, - id); - cachedEntityDao.deleteBase(entityType, id); - } else { - return cachedJson; - } - } catch (Exception e) { - LOG.warn( - "Failed to deserialize cached entity, evicting and falling back to database: {} {}", - entityType, - id, - e); - try { - cachedEntityDao.deleteBase(entityType, id); - } catch (Exception evictError) { - LOG.debug("Failed to evict bad cache entry: {} {}", entityType, id, evictError); - } - } - } - LOG.debug("CACHE MISS: Entity not in Redis cache: {} {}", entityType, id); - } - } - - // Load raw JSON from database - LOG.debug("Loading entity from database: {} {}", entityType, id); - String json = dao.findById(dao.getTableName(), id, dao.getCondition(ALL)); - if (json == null) { - throw new EntityNotFoundException(String.format("Entity not found: %s %s", entityType, id)); - } - - // Validate - EntityInterface entity = JsonUtils.readValue(json, repository.getEntityClass()); - if (!isValidEntityForCache(entity)) { - if (entity.getId() == null) { - LOG.error( - "CRITICAL: Entity loaded from database has null ID! Type: {}, Expected ID: {}, FQN: {}", - entityType, - id, - entity.getFullyQualifiedName()); - entity.setId(id); - json = JsonUtils.pojoToJson(entity); - } - entity = JsonUtils.readValue(json, repository.getEntityClass()); - if (!isValidEntityForCache(entity)) { - LOG.error("Entity from database is invalid for caching: {} {}", entityType, id); - throw new IllegalStateException( - String.format("Invalid entity from database: %s %s", entityType, id)); - } - } - - return json; - } - } - - public static class DescriptionTaskWorkflow extends TaskWorkflow { - DescriptionTaskWorkflow(ThreadContext threadContext) { - super(threadContext); - } - - @Override - public EntityInterface performTask(String user, ResolveTask resolveTask) { - EntityInterface aboutEntity = threadContext.getAboutEntity(); - aboutEntity.setDescription( - org.openmetadata.service.util.DescriptionSanitizer.sanitize(resolveTask.getNewValue())); - return aboutEntity; - } - } - - public static class TagTaskWorkflow extends TaskWorkflow { - TagTaskWorkflow(ThreadContext threadContext) { - super(threadContext); - } - - @Override - public EntityInterface performTask(String user, ResolveTask resolveTask) { - List tags = JsonUtils.readObjects(resolveTask.getNewValue(), TagLabel.class); - EntityInterface aboutEntity = threadContext.getAboutEntity(); - aboutEntity.setTags(tags); - return aboutEntity; - } - } - - /** - * Generic approval task workflow usable for any entity. Checks that the acting user is a - * reviewer of the entity, then delegates resolution to the governance WorkflowHandler. Falls - * back to a direct entityStatus patch when the Flowable workflow record no longer exists (e.g. - * after a corrupted restart). - */ - public static class ApprovalTaskWorkflow extends TaskWorkflow { - ApprovalTaskWorkflow(ThreadContext threadContext) { - super(threadContext); - } - - @Override - public EntityInterface performTask(String user, ResolveTask resolveTask) { - EntityInterface entity = threadContext.getAboutEntity(); - verifyReviewer(entity, user); - - UUID taskId = threadContext.getThread().getId(); - Map variables = new HashMap<>(); - variables.put(RESULT_VARIABLE, resolveTask.getNewValue().equalsIgnoreCase("approved")); - variables.put(UPDATED_BY_VARIABLE, user); - WorkflowHandler workflowHandler = WorkflowHandler.getInstance(); - boolean workflowSuccess = - workflowHandler.resolveLegacyThreadTask( - taskId, workflowHandler.transformToNodeVariables(taskId, variables)); - - if (!workflowSuccess) { - LOG.warn("Workflow failed for taskId='{}', applying status directly", taskId); - Boolean approved = (Boolean) variables.get(RESULT_VARIABLE); - String entityStatus = (approved != null && approved) ? "Approved" : "Rejected"; - EntityFieldUtils.setEntityField( - entity, - threadContext.getAbout().getEntityType(), - user, - FIELD_ENTITY_STATUS, - entityStatus, - true); - } - - return entity; - } - } - - /** - * Checks that {@code user} is an assignee of the given task thread. Throws - * {@link AuthorizationException} if not. - */ - public static void checkUpdatedByTaskAssignee(Thread thread, String user) { - List assignees = listOrEmpty(thread.getTask().getAssignees()); - if (nullOrEmpty(assignees)) { - return; // no assignees configured – allow any user (backward compat) + /** + * Checks that {@code user} is an assignee of the given task thread. Throws + * {@link AuthorizationException} if not. + */ + public static void checkUpdatedByTaskAssignee(Thread thread, String user) { + List assignees = listOrEmpty(thread.getTask().getAssignees()); + if (nullOrEmpty(assignees)) { + return; // no assignees configured – allow any user (backward compat) } boolean isAssignee = assignees.stream() @@ -10555,1182 +9676,162 @@ protected void fetchAndSetFields(List entities, Fields fields) { } } + // --- field-fetch delegators (see BulkFieldFetcher) inserted below --- private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields fields) { - if (nullOrEmpty(entities) || fields == null) { - return Collections.emptySet(); - } - - boolean loadOwners = supportsOwners && fields.contains(FIELD_OWNERS); - boolean loadFollowers = supportsFollower && fields.contains(FIELD_FOLLOWERS); - boolean loadDomains = supportsDomains && fields.contains(FIELD_DOMAINS); - boolean loadReviewers = supportsReviewers && fields.contains(FIELD_REVIEWERS); - boolean loadDataProducts = supportsDataProducts && fields.contains(FIELD_DATA_PRODUCTS); - boolean loadDataContract = supportsDataContract && fields.contains(FIELD_DATA_CONTRACT); - boolean loadVotes = supportsVotes && fields.contains(FIELD_VOTES); - boolean loadChildren = supportsChildren && fields.contains(FIELD_CHILDREN); - boolean loadExperts = supportsExperts && fields.contains(FIELD_EXPERTS); - - if (!loadOwners - && !loadFollowers - && !loadDomains - && !loadReviewers - && !loadDataProducts - && !loadDataContract - && !loadVotes - && !loadChildren - && !loadExperts) { - return Collections.emptySet(); - } - - List incomingRelations = new ArrayList<>(); - if (loadOwners) { - incomingRelations.add(Relationship.OWNS.ordinal()); - } - if (loadFollowers) { - incomingRelations.add(Relationship.FOLLOWS.ordinal()); - } - if (loadDomains || loadDataProducts) { - incomingRelations.add(Relationship.HAS.ordinal()); - } - if (loadReviewers) { - incomingRelations.add(Relationship.REVIEWS.ordinal()); - } - if (loadVotes) { - incomingRelations.add(Relationship.VOTED.ordinal()); - } - - List outgoingRelations = new ArrayList<>(); - if (loadChildren || loadDataContract) { - outgoingRelations.add(Relationship.CONTAINS.ordinal()); - } - if (loadExperts) { - outgoingRelations.add(Relationship.EXPERT.ordinal()); - } - - List entityIds = entityListToStrings(entities); - List incomingRecords = - incomingRelations.isEmpty() - ? Collections.emptyList() - : daoCollection - .relationshipDAO() - .findFromBatchWithRelations(entityIds, entityType, incomingRelations, ALL); - List outgoingRecords = - outgoingRelations.isEmpty() - ? Collections.emptyList() - : daoCollection - .relationshipDAO() - .findToBatchWithRelations(entityIds, entityType, outgoingRelations, ALL); - - Map> incomingRefsByType = - resolveRelationshipEntityReferencesByType(incomingRecords, true); - Map> outgoingRefsByType = - resolveRelationshipEntityReferencesByType(outgoingRecords, false); - - Map> ownersByEntity = loadOwners ? new HashMap<>() : null; - Map> followersByEntity = loadFollowers ? new HashMap<>() : null; - Map> domainsByEntity = loadDomains ? new HashMap<>() : null; - Map> reviewersByEntity = loadReviewers ? new HashMap<>() : null; - Map> dataProductsByEntity = - loadDataProducts ? new HashMap<>() : null; - Map> upVotersByEntity = loadVotes ? new HashMap<>() : null; - Map> downVotersByEntity = loadVotes ? new HashMap<>() : null; - Map> childrenByEntity = loadChildren ? new HashMap<>() : null; - Map dataContractByEntity = loadDataContract ? new HashMap<>() : null; - Map> expertsByEntity = loadExperts ? new HashMap<>() : null; - - for (CollectionDAO.EntityRelationshipObject record : incomingRecords) { - UUID entityId = UUID.fromString(record.getToId()); - UUID sourceId = UUID.fromString(record.getFromId()); - String sourceType = record.getFromEntity(); - EntityReference sourceRef = lookupRelationshipRef(incomingRefsByType, sourceType, sourceId); - if (sourceRef == null) { - continue; - } - - Relationship relationship = relationshipFromOrdinal(record.getRelation()); - if (relationship == null) { - continue; - } - switch (relationship) { - case OWNS -> { - if (loadOwners) { - ownersByEntity.computeIfAbsent(entityId, ignored -> new ArrayList<>()).add(sourceRef); - } - } - case FOLLOWS -> { - if (loadFollowers && USER.equals(sourceType)) { - followersByEntity - .computeIfAbsent(entityId, ignored -> new ArrayList<>()) - .add(sourceRef); - } - } - case HAS -> { - if (loadDomains && DOMAIN.equals(sourceType)) { - domainsByEntity.computeIfAbsent(entityId, ignored -> new ArrayList<>()).add(sourceRef); - } else if (loadDataProducts && DATA_PRODUCT.equals(sourceType)) { - dataProductsByEntity - .computeIfAbsent(entityId, ignored -> new ArrayList<>()) - .add(sourceRef); - } - } - case REVIEWS -> { - if (loadReviewers) { - reviewersByEntity - .computeIfAbsent(entityId, ignored -> new ArrayList<>()) - .add(sourceRef); - } - } - case VOTED -> { - if (loadVotes && USER.equals(sourceType)) { - VoteType voteType = JsonUtils.readValue(record.getJson(), VoteType.class); - if (voteType == VoteType.VOTED_UP) { - upVotersByEntity - .computeIfAbsent(entityId, ignored -> new ArrayList<>()) - .add(sourceRef); - } else if (voteType == VoteType.VOTED_DOWN) { - downVotersByEntity - .computeIfAbsent(entityId, ignored -> new ArrayList<>()) - .add(sourceRef); - } - } - } - default -> { - // no-op - } - } - } - - for (CollectionDAO.EntityRelationshipObject record : outgoingRecords) { - UUID entityId = UUID.fromString(record.getFromId()); - UUID targetId = UUID.fromString(record.getToId()); - String targetType = record.getToEntity(); - EntityReference targetRef = lookupRelationshipRef(outgoingRefsByType, targetType, targetId); - if (targetRef == null) { - continue; - } - - Relationship relationship = relationshipFromOrdinal(record.getRelation()); - if (relationship == null) { - continue; - } - switch (relationship) { - case CONTAINS -> { - if (loadChildren && entityType.equals(targetType)) { - childrenByEntity.computeIfAbsent(entityId, ignored -> new ArrayList<>()).add(targetRef); - } else if (loadDataContract && DATA_CONTRACT.equals(targetType)) { - dataContractByEntity.putIfAbsent(entityId, targetRef); - } - } - case EXPERT -> { - if (loadExperts && USER.equals(targetType)) { - expertsByEntity.computeIfAbsent(entityId, ignored -> new ArrayList<>()).add(targetRef); - } - } - default -> { - // no-op - } - } - } - - Set handledFields = new HashSet<>(); - for (T entity : entities) { - UUID entityId = entity.getId(); - if (loadOwners) { - entity.setOwners(ownersByEntity.getOrDefault(entityId, Collections.emptyList())); - } - if (loadFollowers) { - entity.setFollowers(followersByEntity.getOrDefault(entityId, Collections.emptyList())); - } - if (loadDomains) { - entity.setDomains(domainsByEntity.getOrDefault(entityId, Collections.emptyList())); - } - if (loadReviewers) { - entity.setReviewers(reviewersByEntity.getOrDefault(entityId, Collections.emptyList())); - } - if (loadDataProducts) { - entity.setDataProducts( - dataProductsByEntity.getOrDefault(entityId, Collections.emptyList())); - } - if (loadVotes) { - List upVoters = - upVotersByEntity.getOrDefault(entityId, Collections.emptyList()); - List downVoters = - downVotersByEntity.getOrDefault(entityId, Collections.emptyList()); - entity.setVotes( - new Votes() - .withUpVotes(upVoters.size()) - .withDownVotes(downVoters.size()) - .withUpVoters(upVoters) - .withDownVoters(downVoters)); - } - if (loadChildren) { - entity.setChildren(childrenByEntity.get(entityId)); - } - if (loadDataContract) { - entity.setDataContract(dataContractByEntity.get(entityId)); - } - if (loadExperts) { - entity.setExperts(expertsByEntity.getOrDefault(entityId, Collections.emptyList())); - } - } - - if (loadOwners) { - handledFields.add(FIELD_OWNERS); - } - if (loadFollowers) { - handledFields.add(FIELD_FOLLOWERS); - } - if (loadDomains) { - handledFields.add(FIELD_DOMAINS); - } - if (loadReviewers) { - handledFields.add(FIELD_REVIEWERS); - } - if (loadDataProducts) { - handledFields.add(FIELD_DATA_PRODUCTS); - } - if (loadVotes) { - handledFields.add(FIELD_VOTES); - } - if (loadChildren) { - handledFields.add(FIELD_CHILDREN); - } - if (loadDataContract) { - handledFields.add(FIELD_DATA_CONTRACT); - } - if (loadExperts) { - handledFields.add(FIELD_EXPERTS); - } - return handledFields; + return bulkFieldFetcher.fetchAndSetRelationshipFieldsInBulk(entities, fields); } private Map> resolveRelationshipEntityReferencesByType( List records, boolean fromSide) { - if (records == null || records.isEmpty()) { - return Collections.emptyMap(); - } - - Map> idsByType = new HashMap<>(); - for (CollectionDAO.EntityRelationshipObject record : records) { - String entityTypeForRef = fromSide ? record.getFromEntity() : record.getToEntity(); - String entityId = fromSide ? record.getFromId() : record.getToId(); - if (nullOrEmpty(entityTypeForRef) - || nullOrEmpty(entityId) - || !Entity.hasEntityRepository(entityTypeForRef)) { - continue; - } - idsByType - .computeIfAbsent(entityTypeForRef, ignored -> new HashSet<>()) - .add(UUID.fromString(entityId)); - } - - if (idsByType.isEmpty()) { - return Collections.emptyMap(); - } - - Map> refsByType = new HashMap<>(); - for (Entry> entry : idsByType.entrySet()) { - List refs = - Entity.getEntityReferencesByIds( - entry.getKey(), new ArrayList<>(entry.getValue()), NON_DELETED); - refsByType.put( - entry.getKey(), - refs.stream() - .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a))); - } - return refsByType; + return bulkFieldFetcher.resolveRelationshipEntityReferencesByType(records, fromSide); } private EntityReference lookupRelationshipRef( Map> refsByType, String entityType, UUID id) { - if (refsByType == null || nullOrEmpty(entityType) || id == null) { - return null; - } - Map refs = refsByType.get(entityType); - return refs == null ? null : refs.get(id); + return bulkFieldFetcher.lookupRelationshipRef(refsByType, entityType, id); } private Relationship relationshipFromOrdinal(int relationOrdinal) { - Relationship[] values = Relationship.values(); - return relationOrdinal >= 0 && relationOrdinal < values.length ? values[relationOrdinal] : null; + return bulkFieldFetcher.relationshipFromOrdinal(relationOrdinal); } private void fetchAndSetOwners(List entities, Fields fields) { - if (!fields.contains(FIELD_OWNERS) || !supportsOwners) { - return; - } - Map> ownersMap = batchFetchOwners(entities); - for (T entity : entities) { - entity.setOwners(ownersMap.getOrDefault(entity.getId(), Collections.emptyList())); - } + bulkFieldFetcher.fetchAndSetOwners(entities, fields); } private void fetchAndSetFollowers(List entities, Fields fields) { - if (!fields.contains(FIELD_FOLLOWERS) || !supportsFollower) { - return; - } - Map> followersMap = batchFetchFollowers(entities); - for (T entity : entities) { - entity.setFollowers(followersMap.getOrDefault(entity.getId(), Collections.emptyList())); - } + bulkFieldFetcher.fetchAndSetFollowers(entities, fields); } private void fetchAndSetTags(List entities, Fields fields) { - if (!fields.contains(FIELD_TAGS) || !supportsTags) { - return; - } - - List entityFQNs = - entities.stream().map(EntityInterface::getFullyQualifiedName).toList(); - - Map> tagsMap = batchFetchTags(entityFQNs); - - // Batch fetch all derived tags in ONE query instead of N queries - List allTags = - tagsMap.values().stream().flatMap(List::stream).collect(Collectors.toList()); - Map> derivedTagsMap = TagLabelUtil.batchFetchDerivedTags(allTags); - - for (T entity : entities) { - List entityTags = - tagsMap.getOrDefault(entity.getFullyQualifiedName(), Collections.emptyList()); - entity.setTags(TagLabelUtil.addDerivedTagsWithPreFetched(entityTags, derivedTagsMap)); - } + bulkFieldFetcher.fetchAndSetTags(entities, fields); } private void fetchAndSetDomains(List entities, Fields fields) { - if (!fields.contains(FIELD_DOMAINS) || !supportsDomains) { - return; - } - - Map> domainsMap = batchFetchDomains(entities); - - for (T entity : entities) { - entity.setDomains(domainsMap.getOrDefault(entity.getId(), Collections.emptyList())); - } + bulkFieldFetcher.fetchAndSetDomains(entities, fields); } private void fetchAndSetExtension(List entities, Fields fields) { - if (!fields.contains(FIELD_EXTENSION) - || !supportsExtension - || entities == null - || entities.isEmpty()) { - return; - } - - Map extensionsMap = batchFetchExtensions(entities); - - for (T entity : entities) { - Object extension = extensionsMap.get(entity.getId()); - entity.setExtension(extension); - } + bulkFieldFetcher.fetchAndSetExtension(entities, fields); } protected void fetchAndSetChildren(List entities, Fields fields) { - if (!fields.contains(FIELD_CHILDREN) || entities == null || nullOrEmpty(entities)) { - return; - } - - Map> childrenMap = batchFetchChildren(entities); - - for (T entity : entities) { - entity.setChildren(childrenMap.get(entity.getId())); - } + bulkFieldFetcher.fetchAndSetChildren(entities, fields); } private void fetchAndSetExperts(List entities, Fields fields) { - if (!fields.contains(FIELD_EXPERTS) || !supportsExperts || nullOrEmpty(entities)) { - return; - } - - Map> expertsMap = batchFetchExperts(entities); - - for (T entity : entities) { - entity.setExperts(expertsMap.getOrDefault(entity.getId(), Collections.emptyList())); - } + bulkFieldFetcher.fetchAndSetExperts(entities, fields); } private void fetchAndSetReviewers(List entities, Fields fields) { - if (!fields.contains(FIELD_REVIEWERS) || !supportsReviewers || nullOrEmpty(entities)) { - return; - } - - Map> reviewersMap = batchFetchReviewers(entities); - - for (T entity : entities) { - List reviewers = - reviewersMap.getOrDefault(entity.getId(), Collections.emptyList()); - entity.setReviewers(reviewers); - } + bulkFieldFetcher.fetchAndSetReviewers(entities, fields); } private void fetchAndSetVotes(List entities, Fields fields) { - if (!fields.contains(FIELD_VOTES) || !supportsVotes || nullOrEmpty(entities)) { - return; - } - - Map votesMap = batchFetchVotes(entities); - - for (T entity : entities) { - entity.setVotes(votesMap.getOrDefault(entity.getId(), new Votes())); - } + bulkFieldFetcher.fetchAndSetVotes(entities, fields); } public void enrichEntitiesForAuth(List entities) { - if (entities == null || entities.isEmpty()) return; - Map> ownersMap = batchFetchOwners(entities); - Map> domainsMap = batchFetchDomains(entities); - for (T entity : entities) { - entity.setOwners(ownersMap.getOrDefault(entity.getId(), entity.getOwners())); - entity.setDomains(domainsMap.getOrDefault(entity.getId(), entity.getDomains())); - } + bulkFieldFetcher.enrichEntitiesForAuth(entities); } private void fetchAndSetDataProducts(List entities, Fields fields) { - if (!fields.contains(FIELD_DATA_PRODUCTS) || !supportsDataProducts || nullOrEmpty(entities)) { - return; - } - - Map> dataProductsMap = batchFetchDataProducts(entities); - - for (T entity : entities) { - entity.setDataProducts(dataProductsMap.getOrDefault(entity.getId(), Collections.emptyList())); - } + bulkFieldFetcher.fetchAndSetDataProducts(entities, fields); } private void fetchAndSetCertification(List entities, Fields fields) { - if (!fields.contains(FIELD_CERTIFICATION) || !supportsCertification || nullOrEmpty(entities)) { - return; - } - - Map certificationMap = batchFetchCertification(entities); - - for (T entity : entities) { - entity.setCertification(certificationMap.get(entity.getId())); - } + bulkFieldFetcher.fetchAndSetCertification(entities, fields); } private Map> batchFetchOwners(List entities) { - var ownersMap = new HashMap>(); - - if (entities == null || entities.isEmpty()) { - return ownersMap; - } - // Use Include.ALL to get all relationships including those for soft-deleted entities - // Use the 3-parameter version to find owners of any entity type (equivalent to passing null in - // single entity version) - var records = - daoCollection - .relationshipDAO() - .findFromBatch(entityListToStrings(entities), Relationship.OWNS.ordinal(), ALL); + return bulkFieldFetcher.batchFetchOwners(entities); + } - LOG.debug( - "batchFetchOwners: Found {} owner relationships for {} entities", - records.size(), - entities.size()); - - // Cache UUID conversions to avoid repeated parsing - Map uuidCache = new HashMap<>(); - - // Group records by entity type to batch fetch entity references (with deduplication) - var ownerIdsByType = new HashMap>(); - records.forEach( - rec -> { - var fromEntity = rec.getFromEntity(); - var fromId = uuidCache.computeIfAbsent(rec.getFromId(), UUID::fromString); - ownerIdsByType.computeIfAbsent(fromEntity, k -> new HashSet<>()).add(fromId); - }); + private Map> batchFetchFollowers(List entities) { + return bulkFieldFetcher.batchFetchFollowers(entities); + } - // Batch fetch entity references for each entity type - var ownerRefsByType = new HashMap>(); - ownerIdsByType.forEach( - (entityType, ownerIds) -> { - var ownerRefs = - Entity.getEntityReferencesByIds(entityType, new ArrayList<>(ownerIds), NON_DELETED); - var refMap = - ownerRefs.stream() - .collect(Collectors.toMap(EntityReference::getId, ref -> ref, (a, b) -> a)); - ownerRefsByType.put(entityType, refMap); - }); + private Map batchFetchVotes(List entities) { + return bulkFieldFetcher.batchFetchVotes(entities); + } - // Map owners to entities (reuse cached UUIDs) - records.forEach( - rec -> { - var toId = uuidCache.computeIfAbsent(rec.getToId(), UUID::fromString); - var fromId = uuidCache.get(rec.getFromId()); // Already cached - var fromEntity = rec.getFromEntity(); - - var refMap = ownerRefsByType.get(fromEntity); - if (refMap != null) { - var ownerRef = refMap.get(fromId); - if (ownerRef != null) { - ownersMap.computeIfAbsent(toId, k -> new ArrayList<>()).add(ownerRef); - } - } - }); + private Map> batchFetchDataProducts(List entities) { + return bulkFieldFetcher.batchFetchDataProducts(entities); + } - return ownersMap; + private Map batchFetchCertification(List entities) { + return bulkFieldFetcher.batchFetchCertification(entities); } - private Map> batchFetchFollowers(List entities) { - if (entities == null || entities.isEmpty()) { - return Collections.emptyMap(); - } + private String createTagKey(TagLabel tag) { + return bulkFieldFetcher.createTagKey(tag); + } - List records = - daoCollection - .relationshipDAO() - .findFromBatch( - entityListToStrings(entities), Relationship.FOLLOWS.ordinal(), Include.ALL); + private Set createTagKeySet(List tags) { + return bulkFieldFetcher.createTagKeySet(tags); + } - Map> followersMap = new HashMap<>(); + protected Map> batchFetchTags(List entityFQNs) { + return bulkFieldFetcher.batchFetchTags(entityFQNs); + } - List followerIds = - records.stream() - .map(record -> UUID.fromString(record.getFromId())) - .collect(Collectors.toList()); + private Map> batchFetchDomains(List entities) { + return bulkFieldFetcher.batchFetchDomains(entities); + } - Map followerRefs = - Entity.getEntityReferencesByIds(USER, followerIds, NON_DELETED).stream() - .collect(Collectors.toMap(EntityReference::getId, Function.identity())); - - records.forEach( - record -> { - UUID entityId = UUID.fromString(record.getToId()); - UUID followerId = UUID.fromString(record.getFromId()); - EntityReference followerRef = followerRefs.get(followerId); - if (followerRef != null) { - followersMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(followerRef); - } - }); + private Map> batchFetchReviewers(List entities) { + return bulkFieldFetcher.batchFetchReviewers(entities); + } - return followersMap; + private Map batchFetchExtensions(List entities) { + return bulkFieldFetcher.batchFetchExtensions(entities); } - private Map batchFetchVotes(List entities) { - var votesMap = new HashMap(); - if (entities == null || entities.isEmpty()) { - return votesMap; - } + private Map> batchFetchExperts(List entities) { + return bulkFieldFetcher.batchFetchExperts(entities); + } - var records = - daoCollection - .relationshipDAO() - .findFromBatch( - entityListToStrings(entities), Relationship.VOTED.ordinal(), Entity.USER, ALL); - - var upVoterIds = new HashMap>(); - var downVoterIds = new HashMap>(); - records.forEach( - rec -> { - UUID entityId = UUID.fromString(rec.getToId()); - UUID userId = UUID.fromString(rec.getFromId()); - VoteType type = JsonUtils.readValue(rec.getJson(), VoteType.class); - if (type == VoteType.VOTED_UP) { - upVoterIds.computeIfAbsent(entityId, k -> new ArrayList<>()).add(userId); - } else if (type == VoteType.VOTED_DOWN) { - downVoterIds.computeIfAbsent(entityId, k -> new ArrayList<>()).add(userId); - } - }); + private Map> batchFetchChildren(List entities) { + return bulkFieldFetcher.batchFetchChildren(entities); + } - Set allUserIds = new HashSet<>(); - upVoterIds.values().forEach(allUserIds::addAll); - downVoterIds.values().forEach(allUserIds::addAll); - Map userRefs = - Entity.getEntityReferencesByIds(Entity.USER, new ArrayList<>(allUserIds), NON_DELETED) - .stream() - .collect(Collectors.toMap(EntityReference::getId, Function.identity())); + protected List entityListToStrings(List entities) { + return bulkFieldFetcher.entityListToStrings(entities); + } - for (T entity : entities) { - List up = - upVoterIds.getOrDefault(entity.getId(), Collections.emptyList()).stream() - .map(userRefs::get) - .filter(Objects::nonNull) - .toList(); - List down = - downVoterIds.getOrDefault(entity.getId(), Collections.emptyList()).stream() - .map(userRefs::get) - .filter(Objects::nonNull) - .toList(); - votesMap.put( - entity.getId(), - new Votes() - .withUpVotes(up.size()) - .withDownVotes(down.size()) - .withUpVoters(up) - .withDownVoters(down)); - } + private Iterator> serializeJsons( + List jsons, Fields fields, UriInfo uriInfo) { + return bulkFieldFetcher.serializeJsons(jsons, fields, uriInfo); + } - return votesMap; + protected void setFieldFromMap( + boolean includeField, List entities, Map valueMap, BiConsumer setter) { + bulkFieldFetcher.setFieldFromMap(includeField, entities, valueMap, setter); } - private Map> batchFetchDataProducts(List entities) { - return batchFetchToIdsOneToMany(entities, Relationship.HAS, Entity.DATA_PRODUCT); + protected void setFieldFromMapSingleRelation( + boolean includeField, List entities, Map valueMap, BiConsumer setter) { + bulkFieldFetcher.setFieldFromMapSingleRelation(includeField, entities, valueMap, setter); } - private Map batchFetchCertification(List entities) { - var result = new HashMap(); - if (entities == null || entities.isEmpty() || !supportsCertification) { - return result; - } + protected Map> batchFetchFromIdsManyToOne( + List entities, Relationship relationship, String toEntityType) { + return bulkFieldFetcher.batchFetchFromIdsManyToOne(entities, relationship, toEntityType); + } - long startTime = System.currentTimeMillis(); - - String certClassification = getCertificationClassification(); - if (certClassification == null) { - return result; - } - - // Build FQN hash → entity ID map (batch query uses hashed FQNs for lookup) - Map entityIdByFqnHash = new HashMap<>(); - List fqnList = new ArrayList<>(); - for (T entity : entities) { - fqnList.add(entity.getFullyQualifiedName()); - entityIdByFqnHash.put( - FullyQualifiedName.buildHash(entity.getFullyQualifiedName()), entity.getId()); - } - - List certTags; - try { - certTags = - daoCollection - .tagUsageDAO() - .getCertTagsInternalBatch( - TagLabel.TagSource.CLASSIFICATION.ordinal(), - fqnList, - FullyQualifiedName.buildHash(certClassification) + ".%"); - } catch (Exception e) { - LOG.warn( - "batchFetchCertification: batch query failed, falling back to individual fetch: {}", - e.getMessage()); - for (T entity : entities) { - result.put(entity.getId(), getCertification(entity)); - } - return result; - } - - for (CollectionDAO.TagUsageDAO.TagLabelWithFQNHash tagWithHash : certTags) { - UUID entityId = entityIdByFqnHash.get(tagWithHash.getTargetFQNHash()); - if (entityId == null || result.containsKey(entityId)) { - continue; - } - TagLabel tagLabel = tagWithHash.toTagLabel(); - TagLabelUtil.applyTagCommonFieldsGracefully(tagLabel); - result.put( - entityId, - new AssetCertification() - .withTagLabel(tagLabel) - .withAppliedDate( - tagLabel.getAppliedAt() != null ? tagLabel.getAppliedAt().getTime() : null) - .withExpiryDate( - tagLabel.getMetadata() != null ? tagLabel.getMetadata().getExpiryDate() : null)); - } - - LOG.debug( - "batchFetchCertification: {} entities, {} certs found in {}ms", - entities.size(), - result.size(), - System.currentTimeMillis() - startTime); - - return result; - } - - /** - * Creates a unique key for a TagLabel combining TagFQN and Source for fast Set-based lookups. - * This replaces O(n) stream().anyMatch() operations with O(1) Set.contains() operations. - */ - private String createTagKey(TagLabel tag) { - return tag.getTagFQN() + ":" + tag.getSource(); - } - - /** - * Creates a Set of tag keys from a list of TagLabels for efficient O(1) lookups. - */ - private Set createTagKeySet(List tags) { - return tags.stream().map(this::createTagKey).collect(Collectors.toSet()); - } - - protected Map> batchFetchTags(List entityFQNs) { - if (entityFQNs == null || entityFQNs.isEmpty()) { - return Collections.emptyMap(); - } - - Map> targetHashToTagLabel = - populateTagLabel(listOrEmpty(daoCollection.tagUsageDAO().getTagsInternalBatch(entityFQNs))); - String certClassification = getCertificationClassification(); - return entityFQNs.stream() - .collect( - Collectors.toMap( - Function.identity(), - fqn -> { - String targetFQNHash = FullyQualifiedName.buildHash(fqn); - List tags = - Optional.ofNullable(targetHashToTagLabel.get(targetFQNHash)) - .filter(list -> !list.isEmpty()) - .orElseGet(ArrayList::new); - if (certClassification != null) { - tags.removeIf( - tag -> - certClassification.equals( - FullyQualifiedName.getParentFQN(tag.getTagFQN()))); - } - return tags; - }, - (a, b) -> a)); - } - - private Map> batchFetchDomains(List entities) { - Map> domainsMap = new HashMap<>(); - - if (entities == null || entities.isEmpty()) { - return domainsMap; - } - List records = - daoCollection - .relationshipDAO() - .findFromBatch(entityListToStrings(entities), Relationship.HAS.ordinal(), DOMAIN, ALL); - - // Collect all unique domain IDs first - var domainIds = - records.stream().map(rec -> UUID.fromString(rec.getFromId())).distinct().toList(); - - // Batch fetch all domain entity references - var domainRefs = Entity.getEntityReferencesByIds(DOMAIN, domainIds, ALL); - var domainRefMap = - domainRefs.stream().collect(Collectors.toMap(EntityReference::getId, ref -> ref)); - - for (CollectionDAO.EntityRelationshipObject rec : records) { - UUID toId = UUID.fromString(rec.getToId()); - UUID fromId = UUID.fromString(rec.getFromId()); - EntityReference domainRef = domainRefMap.get(fromId); - domainsMap.computeIfAbsent(toId, k -> new ArrayList<>()).add(domainRef); - } - - return domainsMap; - } - - private Map> batchFetchReviewers(List entities) { - if (entities == null || entities.isEmpty()) { - return new HashMap<>(); - } - - // Use Include.ALL to get all relationships including those for soft-deleted entities - var records = - daoCollection - .relationshipDAO() - .findFromBatch(entityListToStrings(entities), Relationship.REVIEWS.ordinal(), ALL); - - var reviewersMap = new HashMap>(); - - // Cache UUID conversions to avoid repeated parsing - Map uuidCache = new HashMap<>(); - - // Group records by entity type to batch fetch entity references (with deduplication) - var reviewerIdsByType = new HashMap>(); - records.forEach( - rec -> { - var fromEntity = rec.getFromEntity(); - var fromId = uuidCache.computeIfAbsent(rec.getFromId(), UUID::fromString); - reviewerIdsByType.computeIfAbsent(fromEntity, k -> new HashSet<>()).add(fromId); - }); - - // Batch fetch entity references for each entity type - var reviewerRefsByType = new HashMap>(); - reviewerIdsByType.forEach( - (entityType, reviewerIds) -> { - var reviewerRefs = - Entity.getEntityReferencesByIds( - entityType, new ArrayList<>(reviewerIds), NON_DELETED); - var refMap = - reviewerRefs.stream() - .collect(Collectors.toMap(EntityReference::getId, ref -> ref, (a, b) -> a)); - reviewerRefsByType.put(entityType, refMap); - }); - - // Map reviewers to entities (reuse cached UUIDs) - records.forEach( - rec -> { - var entityId = uuidCache.computeIfAbsent(rec.getToId(), UUID::fromString); - var fromId = uuidCache.get(rec.getFromId()); // Already cached - var fromEntity = rec.getFromEntity(); - - var refMap = reviewerRefsByType.get(fromEntity); - if (refMap != null) { - var reviewerRef = refMap.get(fromId); - if (reviewerRef != null) { - reviewersMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(reviewerRef); - } - } - }); - - return reviewersMap; - } - - private Map batchFetchExtensions(List entities) { - if (!supportsExtension || entities == null || entities.isEmpty()) { - return Collections.emptyMap(); - } - String fieldFQNPrefix = TypeRegistry.getCustomPropertyFQNPrefix(entityType); - - List records = - daoCollection - .entityExtensionDAO() - .getExtensionsBatch(entityListToStrings(entities), fieldFQNPrefix); - - Map> extensionsMap = - records.stream().collect(Collectors.groupingBy(CollectionDAO.ExtensionRecordWithId::id)); - - Map result = new HashMap<>(); - - for (Entry> entry : extensionsMap.entrySet()) { - UUID entityId = entry.getKey(); - List extensionRecords = entry.getValue(); - - ObjectNode objectNode = JsonUtils.getObjectNode(); - for (CollectionDAO.ExtensionRecordWithId record : extensionRecords) { - String fieldName = TypeRegistry.getPropertyName(record.extensionName()); - JsonNode extensionJsonNode = JsonUtils.readTree(record.extensionJson()); - objectNode.set(fieldName, extensionJsonNode); - } - - result.put(entityId, objectNode); - } - - return result; - } - - private Map> batchFetchExperts(List entities) { - if (!supportsExperts || nullOrEmpty(entities)) { - return Collections.emptyMap(); - } - - // Batch fetch all expert relationships - experts are TO relationships - List records = - daoCollection - .relationshipDAO() - .findToBatch(entityListToStrings(entities), Relationship.EXPERT.ordinal(), USER); - - Map> expertsMap = new HashMap<>(); - - // Cache UUID conversions to avoid repeated parsing - Map uuidCache = new HashMap<>(); - - // findToBatch returns fromId=entity, toId=user — collect user IDs from toId - List expertIds = - records.stream() - .map(record -> uuidCache.computeIfAbsent(record.getToId(), UUID::fromString)) - .distinct() - .collect(Collectors.toList()); - - // Batch fetch all expert references, filtering out soft-deleted users - Map expertRefs = - Entity.getEntityReferencesByIds(USER, expertIds, NON_DELETED).stream() - .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); - - // Group experts by entity - records.forEach( - record -> { - UUID entityId = uuidCache.computeIfAbsent(record.getFromId(), UUID::fromString); - UUID expertId = uuidCache.get(record.getToId()); // Already cached above - EntityReference expertRef = expertRefs.get(expertId); - if (expertRef != null) { - expertsMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(expertRef); - } - }); - - LOG.debug( - "batchFetchExperts: Found {} expert relationships for {} entities", - records.size(), - entities.size()); - - return expertsMap; - } - - private Map> batchFetchChildren(List entities) { - if (entities == null || entities.isEmpty()) { - return new HashMap<>(); - } - - // Use Include.ALL to get all relationships including those for soft-deleted entities - var records = - daoCollection - .relationshipDAO() - .findToBatch( - entityListToStrings(entities), Relationship.CONTAINS.ordinal(), entityType, ALL); - - var childrenMap = new HashMap>(); - - if (CollectionUtils.isEmpty(records)) { - return childrenMap; - } - - var idReferenceMap = - Entity.getEntityReferencesByIds( - records.get(0).getToEntity(), - records.stream().map(e -> UUID.fromString(e.getToId())).distinct().toList(), - ALL) - .stream() - .collect(Collectors.toMap(e -> e.getId().toString(), Function.identity())); - - records.forEach( - rec -> { - var entityId = UUID.fromString(rec.getFromId()); - var childrenRef = idReferenceMap.get(rec.getToId()); - if (childrenRef != null) { - childrenMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(childrenRef); - } - }); - - return childrenMap; - } - - List entityListToStrings(List entities) { - return entities.stream().map(EntityInterface::getId).map(UUID::toString).toList(); - } - - private Iterator> serializeJsons( - List jsons, Fields fields, UriInfo uriInfo) { - List> results = new ArrayList<>(); - List entities = new ArrayList<>(); - - for (String json : jsons) { - try { - T entity = JsonUtils.readValue(json, entityClass); - entities.add(entity); - } catch (Exception e) { - EntityError entityError = - new EntityError() - .withMessage("Failed to deserialize entity: " + e.getMessage()) - .withEntity(null); - results.add(Either.right(entityError)); - } - } - - if (!entities.isEmpty()) { - try { - setFieldsInBulk(fields, entities); - if (!nullOrEmpty(uriInfo)) { - entities.forEach(entity -> withHref(uriInfo, entity)); - } - - for (T entity : entities) { - results.add(Either.left(entity)); - } - } catch (Exception e) { - LOG.warn("setFieldsInBulk failed in serializeJsons, falling back to per-entity loading", e); - for (T entity : entities) { - try { - setFieldsInternal(entity, fields); - setInheritedFields(entity, fields); - clearFieldsInternal(entity, fields); - if (!nullOrEmpty(uriInfo)) { - entity = withHref(uriInfo, entity); - } - results.add(Either.left(entity)); - } catch (Exception individualError) { - clearFieldsInternal(entity, fields); - EntityError entityError = - new EntityError().withMessage(individualError.getMessage()).withEntity(entity); - results.add(Either.right(entityError)); - } - } - } - } - return results.iterator(); - } - - protected void setFieldFromMap( - boolean includeField, List entities, Map valueMap, BiConsumer setter) { - if (!includeField || entities.isEmpty()) { - return; - } - for (T entity : entities) { - V value = valueMap.get(entity.getId()); - setter.accept(entity, value); - } - } - - protected void setFieldFromMapSingleRelation( - boolean includeField, List entities, Map valueMap, BiConsumer setter) { - if (!includeField || entities.isEmpty()) { - return; - } - for (T entity : entities) { - V value = valueMap.get(entity.getId()); - setter.accept(entity, value); - } - } - - protected Map> batchFetchFromIdsManyToOne( - List entities, Relationship relationship, String toEntityType) { - var resultMap = new HashMap>(); - if (entities == null || entities.isEmpty()) { - return resultMap; - } - - // Use Include.ALL to get all relationships including those for soft-deleted entities - var records = - daoCollection - .relationshipDAO() - .findToBatch(entityListToStrings(entities), relationship.ordinal(), toEntityType, ALL); - - var idReferenceMap = - Entity.getEntityReferencesByIds( - toEntityType, - records.stream().map(e -> UUID.fromString(e.getToId())).distinct().toList(), - ALL) - .stream() - .collect(Collectors.toMap(e -> e.getId().toString(), Function.identity())); - - records.forEach( - record -> { - var entityId = UUID.fromString(record.getFromId()); - var relatedRef = idReferenceMap.get(record.getToId()); - if (relatedRef != null) { - resultMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(relatedRef); - } - }); - - return resultMap; - } - - protected Map> batchFetchToIdsOneToMany( - List entities, Relationship relationship, String fromEntityType) { - var resultMap = new HashMap>(); - if (entities == null || entities.isEmpty()) { - return resultMap; - } - - // Use Include.ALL to get all relationships including those for soft-deleted entities - var records = - daoCollection - .relationshipDAO() - .findFromBatch( - entityListToStrings(entities), relationship.ordinal(), fromEntityType, ALL); - - var idReferenceMap = - Entity.getEntityReferencesByIds( - fromEntityType, - records.stream().map(e -> UUID.fromString(e.getFromId())).distinct().toList(), - ALL) - .stream() - .collect(Collectors.toMap(e -> e.getId().toString(), Function.identity())); - - records.forEach( - record -> { - var entityId = UUID.fromString(record.getToId()); - var relatedRef = idReferenceMap.get(record.getFromId()); - if (relatedRef != null) { - resultMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(relatedRef); - } - }); - - return resultMap; - } + protected Map> batchFetchToIdsOneToMany( + List entities, Relationship relationship, String fromEntityType) { + return bulkFieldFetcher.batchFetchToIdsOneToMany(entities, relationship, fromEntityType); + } protected Map batchFetchFromIdsAndRelationSingleRelation( List entities, Relationship relationship) { - var resultMap = new HashMap(); - if (entities == null || entities.isEmpty()) { - return resultMap; - } - - // Use Include.ALL to get all relationships including those for soft-deleted entities - var records = - daoCollection - .relationshipDAO() - .findFromBatch(entityListToStrings(entities), relationship.ordinal(), ALL); - - var idReferenceMap = new HashMap(); - - // Group by entity type to make efficient batch calls - var entityTypeToIds = - records.stream() - .collect( - Collectors.groupingBy( - CollectionDAO.EntityRelationshipObject::getFromEntity, - Collectors.mapping( - CollectionDAO.EntityRelationshipObject::getFromId, Collectors.toList()))); - - entityTypeToIds.forEach( - (entityType, idStrings) -> { - var ids = idStrings.stream().map(UUID::fromString).distinct().toList(); - var refs = Entity.getEntityReferencesByIds(entityType, ids, ALL); - refs.forEach(ref -> idReferenceMap.put(ref.getId().toString(), ref)); - }); - - records.forEach( - record -> { - var entityId = UUID.fromString(record.getToId()); - var relatedRef = idReferenceMap.get(record.getFromId()); - if (relatedRef != null) { - resultMap.put(entityId, relatedRef); - } - }); - - return resultMap; + return bulkFieldFetcher.batchFetchFromIdsAndRelationSingleRelation(entities, relationship); } - /** Bulk populate field tags for multiple entities using chunked exact-match IN on field FQN hashes. */ protected void bulkPopulateEntityFieldTags( List entities, java.util.function.Function> fieldExtractor) { - - if (entities == null || entities.isEmpty()) { - return; - } - - Set fieldFQNs = new LinkedHashSet<>(); - Map> flatFieldsByEntity = new HashMap<>(); - for (T entity : entities) { - List fields = fieldExtractor.apply(entity); - if (fields != null) { - List flattenedFields = EntityUtil.getFlattenedEntityField(fields); - flatFieldsByEntity.put(entity, flattenedFields); - for (F field : listOrEmpty(flattenedFields)) { - if (field.getFullyQualifiedName() != null) { - fieldFQNs.add(field.getFullyQualifiedName()); - } - } - } - } - - if (fieldFQNs.isEmpty()) { - return; - } - - // Fetch tags in chunked IN queries, then enrich once - List fieldFQNList = new ArrayList<>(fieldFQNs); - int batchSize = 5000; - List tagUsages = new ArrayList<>(); - for (int i = 0; i < fieldFQNList.size(); i += batchSize) { - List chunk = fieldFQNList.subList(i, Math.min(i + batchSize, fieldFQNList.size())); - tagUsages.addAll(listOrEmpty(daoCollection.tagUsageDAO().getTagsInternalBatch(chunk))); - } - Map> tagsByFieldHash = populateTagLabel(tagUsages); - - Map> derivedTagsMap; - try { - List tagLabels = - tagsByFieldHash.values().stream().flatMap(List::stream).collect(Collectors.toList()); - derivedTagsMap = TagLabelUtil.batchFetchDerivedTags(tagLabels); - } catch (Exception ex) { - LOG.warn("Failed to batch fetch derived tags for fields. Skipping derived tags.", ex); - derivedTagsMap = Collections.emptyMap(); - } - - for (T entity : entities) { - List flattenedFields = flatFieldsByEntity.get(entity); - if (flattenedFields != null) { - for (F field : listOrEmpty(flattenedFields)) { - String fieldHash = FullyQualifiedName.buildHash(field.getFullyQualifiedName()); - List fieldTags = tagsByFieldHash.get(fieldHash); - if (fieldTags == null) { - field.setTags(new ArrayList<>()); - } else { - field.setTags(TagLabelUtil.addDerivedTagsWithPreFetched(fieldTags, derivedTagsMap)); - } - } - } - } + bulkFieldFetcher.bulkPopulateEntityFieldTags(entities, fieldExtractor); } protected List entityListToUUID(List entities) { @@ -11799,68 +9900,11 @@ public void createChangeEventForBulkOperation( daoCollection.changeEventDAO().insert(JsonUtils.pojoToJson(changeEvent)); } + // --- bulk-import delegators (see BulkImportService) inserted below --- private CsvImportResult createLeanCsvImportResult(CsvImportResult fullResult) { - CsvImportResult leanResult = - new CsvImportResult() - .withDryRun(fullResult.getDryRun()) - .withStatus(fullResult.getStatus()) - .withNumberOfRowsProcessed(fullResult.getNumberOfRowsProcessed()) - .withNumberOfRowsPassed(fullResult.getNumberOfRowsPassed()) - .withNumberOfRowsFailed(fullResult.getNumberOfRowsFailed()) - .withAbortReason(fullResult.getAbortReason()); - - if (nullOrEmpty(fullResult.getImportResultsCsv())) { - return leanResult; - } - - StringWriter stringWriter = new StringWriter(); - try (CSVParser parser = - CSVParser.parse( - fullResult.getImportResultsCsv(), CSVFormat.DEFAULT.withFirstRecordAsHeader())) { - int nameIndex = -1; - List headerNames = parser.getHeaderNames(); - for (int i = 0; i < headerNames.size(); i++) { - if (headerNames.get(i).toLowerCase().contains("name")) { - nameIndex = i; - break; - } - } - - String[] leanHeaders = {"status", "details", "name"}; - try (CSVPrinter printer = - new CSVPrinter(stringWriter, CSVFormat.DEFAULT.withHeader(leanHeaders))) { - for (CSVRecord record : parser) { - String name = (nameIndex != -1 && nameIndex < record.size()) ? record.get(nameIndex) : ""; - printer.printRecord(record.get("status"), record.get("details"), name); - } - } - leanResult.setImportResultsCsv(stringWriter.toString()); - } catch (IOException | IllegalArgumentException e) { - // If parsing fails, just return the original CSV to avoid losing data - LOG.warn("Failed to create lean CSV for change description, returning full CSV", e); - leanResult.setImportResultsCsv(fullResult.getImportResultsCsv()); - } - return leanResult; + return bulkImportService.createLeanCsvImportResult(fullResult); } - private static final ConcurrentHashMap> BULK_JOBS = - new ConcurrentHashMap<>(); - - // Cached metrics to avoid Timer.builder overhead on every call - private static final ConcurrentHashMap ENTITY_LATENCY_TIMERS = - new ConcurrentHashMap<>(); - private static final ConcurrentHashMap ENTITY_QUEUE_WAIT_TIMERS = - new ConcurrentHashMap<>(); - private static final ConcurrentHashMap BULK_OPERATION_TIMERS = - new ConcurrentHashMap<>(); - private static final ConcurrentHashMap BATCH_SIZE_SUMMARIES = - new ConcurrentHashMap<>(); - private static final ConcurrentHashMap SUCCESS_RATE_SUMMARIES = - new ConcurrentHashMap<>(); - - private static final int MAX_CONCURRENT_BULK_JOBS = 100; - private static final Semaphore BULK_JOB_PERMITS = new Semaphore(MAX_CONCURRENT_BULK_JOBS); - public CompletableFuture submitAsyncBulkOperation( UriInfo uriInfo, List entities, @@ -11869,640 +9913,28 @@ public CompletableFuture submitAsyncBulkOperation( boolean overrideMetadata, List authFailedResponses, int totalRequests) { - - // Acquire a permit before scheduling — Semaphore is thread-safe and avoids TOCTOU races - if (!BULK_JOB_PERMITS.tryAcquire()) { - throw new jakarta.ws.rs.WebApplicationException( - "Too many concurrent bulk jobs (max " + MAX_CONCURRENT_BULK_JOBS + "). Retry later.", - jakarta.ws.rs.core.Response.Status.TOO_MANY_REQUESTS); - } - - String jobId = UUID.randomUUID().toString(); - LOG.info( - "Submitting async bulk operation with jobId: {} for {} entities", jobId, entities.size()); - - CompletableFuture job; - try { - job = - CompletableFuture.supplyAsync( - () -> { - try { - return bulkCreateOrUpdateEntitiesSequential( - uriInfo, entities, userName, existingByFqn, overrideMetadata); - } catch (Exception e) { - LOG.error("Async bulk operation failed for jobId: {}", jobId, e); - BulkOperationResult errorResult = new BulkOperationResult(); - errorResult.setStatus(ApiStatus.FAILURE); - errorResult.setNumberOfRowsFailed(entities.size()); - errorResult.setNumberOfRowsPassed(0); - return errorResult; - } - }, - BulkExecutor.getInstance().getExecutor()); - } catch (Exception e) { - BULK_JOB_PERMITS.release(); - throw e; - } - - // Merge auth failures into the final result so polling clients see the complete picture - CompletableFuture mergedJob = - job.thenApply( - result -> { - if (!authFailedResponses.isEmpty()) { - result.setNumberOfRowsFailed( - result.getNumberOfRowsFailed() + authFailedResponses.size()); - result.setNumberOfRowsProcessed(totalRequests); - if (result.getFailedRequest() == null) { - result.setFailedRequest(new ArrayList<>(authFailedResponses)); - } else { - result.getFailedRequest().addAll(authFailedResponses); - } - if (result.getNumberOfRowsPassed() > 0) { - result.setStatus(ApiStatus.PARTIAL_SUCCESS); - } else { - result.setStatus(ApiStatus.FAILURE); - } - } - return result; - }); - - BULK_JOBS.put(jobId, mergedJob); - - mergedJob.whenComplete( - (result, throwable) -> { - BULK_JOB_PERMITS.release(); - CompletableFuture.delayedExecutor(5, TimeUnit.MINUTES) - .execute(() -> BULK_JOBS.remove(jobId)); - }); - - return mergedJob; - } - - /** - * Fields used to hydrate inherited relationships for changed entities during bulk updates. - * - *

Use PUT-update fields as baseline and explicitly include inheritable fields so repositories - * with inheritance beyond owners/domains (for example retentionPeriod or reviewers) keep behavior - * intact without loading every allowed field. - */ - protected Fields getBulkUpdateInheritanceFields() { - Set bulkFields = new HashSet<>(putFields.getFieldList()); - String inheritableFields = getInheritableFields(); - if (!nullOrEmpty(inheritableFields)) { - for (String field : inheritableFields.split(",")) { - String normalized = field.trim(); - if (!normalized.isEmpty() && allowedFields.contains(normalized)) { - bulkFields.add(normalized); - } - } - } - return new Fields(allowedFields, bulkFields); - } - - /** - * Returns true when a connector-supplied entity is provably unchanged from what is stored, so - * the bulk update path can skip field hydration and per-field diffing. Requires a non-empty - * sourceHash on both the incoming entity and the stored original that match, an existing - * non-deleted original, and that the FQN appears only once in the batch (duplicate FQNs need - * the full updater path so each occurrence diffs against a fresh snapshot). - */ - private boolean isSourceHashUnchanged( - T entity, Map hydratedOriginalByFqn, Map updateFrequencyByFqn) { - String fqn = entity.getFullyQualifiedName(); - if (nullOrEmpty(fqn) || updateFrequencyByFqn.getOrDefault(fqn, 0) > 1) { - return false; - } - String incomingHash = entity.getSourceHash(); - if (nullOrEmpty(incomingHash)) { - return false; - } - T original = hydratedOriginalByFqn.get(fqn); - if (original == null || Boolean.TRUE.equals(original.getDeleted())) { - return false; - } - return incomingHash.equals(original.getSourceHash()); - } - - private void bulkUpdateEntities( - UriInfo uriInfo, - List updateEntities, - Map existingByFqn, - String userName, - boolean overrideMetadata, - List successRequests, - List failedRequests, - List entityLatenciesNanos) { - - if (updateEntities.isEmpty()) return; - - long batchStartTime = System.nanoTime(); - - // Batch load fields once per unique FQN. Duplicate rows in the same bulk request - // should reuse a hydrated original snapshot. - Map hydratedOriginalByFqn = new LinkedHashMap<>(); - Map updateFrequencyByFqn = new HashMap<>(); - for (T entity : updateEntities) { - String fqn = entity.getFullyQualifiedName(); - if (nullOrEmpty(fqn)) { - continue; - } - updateFrequencyByFqn.merge(fqn, 1, Integer::sum); - T original = existingByFqn.get(fqn); - if (original != null) { - hydratedOriginalByFqn.putIfAbsent(fqn, original); - } - } - - // sourceHash fast-path: skip entities whose connector-supplied sourceHash matches the - // stored value. This avoids field hydration and per-field diffing for unchanged entities. - // A skipped entity is reported as a no-change success - identical to the outcome of a full - // diff that finds nothing changed - so callers see no behavioral difference. Disabled when - // overrideMetadata is set: the caller explicitly wants stored metadata overwritten now, so a - // matching sourceHash must not short-circuit that. - List entitiesToProcess = new ArrayList<>(); - for (T entity : updateEntities) { - if (!overrideMetadata - && isSourceHashUnchanged(entity, hydratedOriginalByFqn, updateFrequencyByFqn)) { - successRequests.add( - new BulkResponse() - .withRequest(entity.getFullyQualifiedName()) - .withStatus(Status.OK.getStatusCode())); - entityLatenciesNanos.add(0L); - recordEntityMetrics(entityType, 0L, 0, true); - } else { - entitiesToProcess.add(entity); - } - } - if (entitiesToProcess.isEmpty()) return; - - // Hydrate only the originals of entities that still need a full diff. - Map originalsToHydrateByFqn = new LinkedHashMap<>(); - for (T entity : entitiesToProcess) { - T original = hydratedOriginalByFqn.get(entity.getFullyQualifiedName()); - if (original != null) { - originalsToHydrateByFqn.putIfAbsent(entity.getFullyQualifiedName(), original); - } - } - List originalsForHydration = new ArrayList<>(originalsToHydrateByFqn.values()); - try { - setFieldsInBulk(putFields, originalsForHydration); - } catch (Exception e) { - LOG.error("setFieldsInBulk failed, marking all updates as failed", e); - for (T entity : entitiesToProcess) { - failedRequests.add( - new BulkResponse() - .withRequest(entity.getFullyQualifiedName()) - .withStatus(Status.BAD_REQUEST.getStatusCode()) - .withMessage("Batch field loading failed: " + e.getMessage())); - } - return; - } - - // Per-entity updater (relationships + change description) - List updaters = new ArrayList<>(); - - try (var ignored = phase("entityUpdaters")) { - for (T entity : entitiesToProcess) { - try { - String fqn = entity.getFullyQualifiedName(); - T hydratedOriginal = hydratedOriginalByFqn.get(fqn); - if (hydratedOriginal == null) { - failedRequests.add( - new BulkResponse() - .withRequest(fqn) - .withStatus(Status.BAD_REQUEST.getStatusCode()) - .withMessage("Entity does not exist")); - continue; - } - T original = - updateFrequencyByFqn.getOrDefault(fqn, 0) > 1 - ? JsonUtils.deepCopy(hydratedOriginal, entityClass) - : hydratedOriginal; - entity.setUpdatedBy(userName); - entity.setUpdatedAt(System.currentTimeMillis()); - - if (Boolean.TRUE.equals(original.getDeleted())) { - restoreEntity(entity.getUpdatedBy(), original.getId()); - } - - EntityUpdater updater = getUpdater(original, entity, Operation.PUT, null); - updater.setOverrideMetadata(overrideMetadata); - updater.updateWithDeferredStore(); - updaters.add(updater); - } catch (Exception e) { - failedRequests.add( - new BulkResponse() - .withRequest(entity.getFullyQualifiedName()) - .withStatus(Status.BAD_REQUEST.getStatusCode()) - .withMessage(e.getMessage())); - } - } - } - - if (updaters.isEmpty()) return; - List changedUpdaters = - updaters.stream() - .filter(updater -> updater.isVersionChanged() || updater.isEntityChanged()) - .toList(); - Fields bulkInheritanceFields = getBulkUpdateInheritanceFields(); - - // Batch DB writes - try { - try (var ignored = phase("batchDbWrites")) { - // Batch version history inserts - List historyIds = new ArrayList<>(); - List historyExtensions = new ArrayList<>(); - List historyJsons = new ArrayList<>(); - for (EntityUpdater updater : changedUpdaters) { - if (updater.isVersionChanged()) { - historyIds.add(updater.getOriginal().getId()); - historyExtensions.add( - EntityUtil.getVersionExtension(entityType, updater.getOriginal().getVersion())); - historyJsons.add(JsonUtils.pojoToJson(updater.getOriginal())); - } - } - if (!historyIds.isEmpty()) { - daoCollection - .entityExtensionDAO() - .insertMany(historyIds, historyExtensions, entityType, historyJsons); - } - - // Batch entity row updates - List entitiesToStore = new ArrayList<>(); - for (EntityUpdater updater : changedUpdaters) { - entitiesToStore.add(updater.getUpdated()); - } - if (!entitiesToStore.isEmpty()) { - updateMany(entitiesToStore); - } - } - - List changedEntities = changedUpdaters.stream().map(EntityUpdater::getUpdated).toList(); - if (!changedEntities.isEmpty()) { - try (var ignored = phase("setInheritedFields")) { - // Only changed entities need inheritance hydration for downstream side effects. - setInheritedFields(changedEntities, bulkInheritanceFields); - } - try (var ignored = phase("invalidateCacheBulk")) { - invalidateMany(changedEntities); - } - try (var ignored = phase("postUpdateEvents")) { - List changeEventJsons = new ArrayList<>(); - for (var updater : changedUpdaters) { - postUpdate(updater.getOriginal(), updater.getUpdated()); - updater.runDeferredReactOperations(); - var changeType = updater.incrementalFieldsChanged() ? ENTITY_UPDATED : ENTITY_NO_CHANGE; - buildChangeEventJsonForBulkOperation(updater.getUpdated(), changeType, userName) - .ifPresent(changeEventJsons::add); - } - insertChangeEventsBatch(changeEventJsons); - } - } - - // Per-entity success + metrics (includes no-change updates). - long batchDuration = System.nanoTime() - batchStartTime; - long perEntityDuration = batchDuration / updaters.size(); - for (var updater : updaters) { - entityLatenciesNanos.add(perEntityDuration); - recordEntityMetrics(entityType, perEntityDuration, 0, true); - successRequests.add( - new BulkResponse() - .withRequest(updater.getUpdated().getFullyQualifiedName()) - .withStatus(Status.OK.getStatusCode())); - } - } catch (Exception batchError) { - LOG.warn("Batch update store failed, falling back to per-entity updates", batchError); - List succeededUpdaters = new ArrayList<>(); - for (var updater : updaters) { - try { - if (updater.isVersionChanged() || updater.isEntityChanged()) { - updater.storeUpdate(); - invalidate(updater.getUpdated()); - } - succeededUpdaters.add(updater); - } catch (Exception e) { - failedRequests.add( - new BulkResponse() - .withRequest(updater.getUpdated().getFullyQualifiedName()) - .withStatus(Status.BAD_REQUEST.getStatusCode()) - .withMessage(e.getMessage())); - } - } - if (!succeededUpdaters.isEmpty()) { - List fallbackChanged = - succeededUpdaters.stream() - .filter(updater -> updater.isVersionChanged() || updater.isEntityChanged()) - .map(EntityUpdater::getUpdated) - .toList(); - if (!fallbackChanged.isEmpty()) { - try (var ignored = phase("invalidateCacheBulk")) { - invalidateMany(fallbackChanged); - } - setInheritedFields(fallbackChanged, bulkInheritanceFields); - } - - List changeEventJsons = new ArrayList<>(); - for (var updater : succeededUpdaters) { - if (updater.isVersionChanged() || updater.isEntityChanged()) { - postUpdate(updater.getOriginal(), updater.getUpdated()); - updater.runDeferredReactOperations(); - var changeType = updater.incrementalFieldsChanged() ? ENTITY_UPDATED : ENTITY_NO_CHANGE; - buildChangeEventJsonForBulkOperation(updater.getUpdated(), changeType, userName) - .ifPresent(changeEventJsons::add); - } - successRequests.add( - new BulkResponse() - .withRequest(updater.getUpdated().getFullyQualifiedName()) - .withStatus(Status.OK.getStatusCode())); - } - insertChangeEventsBatch(changeEventJsons); - } - } - } - - private BulkOperationResult bulkCreateOrUpdateEntitiesSequential( - UriInfo uriInfo, - List entities, - String userName, - Map existingByFqn, - boolean overrideMetadata) { - - BulkOperationResult result = new BulkOperationResult(); - result.setStatus(ApiStatus.SUCCESS); - - List successRequests = new ArrayList<>(); - List failedRequests = new ArrayList<>(); - - long bulkStartTime = System.nanoTime(); - List entityLatenciesNanos = new ArrayList<>(); - - // Separate into creates and updates using the pre-fetched map - // For duplicate FQNs within the batch, first occurrence goes to creates, - // subsequent occurrences go to updates (processed after creates) - List newEntities = new ArrayList<>(); - List updateEntities = new ArrayList<>(); - Set seenNewFqns = new HashSet<>(); - for (T entity : entities) { - String fqn = entity.getFullyQualifiedName(); - if (existingByFqn.containsKey(fqn)) { - updateEntities.add(entity); - } else if (seenNewFqns.contains(fqn)) { - updateEntities.add(entity); - } else { - seenNewFqns.add(fqn); - newEntities.add(entity); - } - } - - // Batch create new entities - if (!newEntities.isEmpty()) { - long batchStartTime = System.nanoTime(); - try { - createManyEntities(newEntities); - long batchDuration = System.nanoTime() - batchStartTime; - long perEntityDuration = batchDuration / newEntities.size(); - List createdChangeEventJsons = new ArrayList<>(); - for (T entity : newEntities) { - entityLatenciesNanos.add(perEntityDuration); - recordEntityMetrics(entityType, perEntityDuration, 0, true); - successRequests.add( - new BulkResponse() - .withRequest(entity.getFullyQualifiedName()) - .withStatus(Status.OK.getStatusCode())); - buildChangeEventJsonForBulkOperation(entity, ENTITY_CREATED, userName) - .ifPresent(createdChangeEventJsons::add); - } - insertChangeEventsBatch(createdChangeEventJsons); - } catch (Exception batchError) { - LOG.warn("Batch create failed, falling back to per-entity creates", batchError); - for (T entity : newEntities) { - long entityStartTime = System.nanoTime(); - try { - PutResponse putResponse = createOrUpdate(uriInfo, entity, userName); - long entityDuration = System.nanoTime() - entityStartTime; - entityLatenciesNanos.add(entityDuration); - recordEntityMetrics(entityType, entityDuration, 0, true); - successRequests.add( - new BulkResponse() - .withRequest(entity.getFullyQualifiedName()) - .withStatus(Status.OK.getStatusCode())); - createChangeEventForBulkOperation( - putResponse.getEntity(), putResponse.getChangeType(), userName); - } catch (Exception e) { - long entityDuration = System.nanoTime() - entityStartTime; - entityLatenciesNanos.add(entityDuration); - if (isDuplicateKeyException(e)) { - LOG.debug( - "Entity already exists (duplicate key), treating as success: {}", - entity.getFullyQualifiedName()); - recordEntityMetrics(entityType, entityDuration, 0, true); - successRequests.add( - new BulkResponse() - .withRequest(entity.getFullyQualifiedName()) - .withStatus(Status.OK.getStatusCode())); - } else { - recordEntityMetrics(entityType, entityDuration, 0, false); - failedRequests.add( - new BulkResponse() - .withRequest(entity.getFullyQualifiedName()) - .withStatus(Status.BAD_REQUEST.getStatusCode()) - .withMessage(e.getMessage())); - } - } - } - } - } - - // For duplicate FQNs within the batch, refresh existingByFqn with newly created entities - if (!updateEntities.isEmpty()) { - List updateFqns = - updateEntities.stream() - .map(T::getFullyQualifiedName) - .filter(fqn -> !existingByFqn.containsKey(fqn)) - .distinct() - .collect(Collectors.toList()); - if (!updateFqns.isEmpty()) { - List newlyCreated = dao.findEntityByNames(updateFqns, Include.ALL); - for (T created : newlyCreated) { - existingByFqn.put(created.getFullyQualifiedName(), created); - } - } - - // Filter out entities whose original doesn't exist (e.g., duplicate FQN whose - // first occurrence failed to create). These can't be updated — report as failed. - Iterator it = updateEntities.iterator(); - while (it.hasNext()) { - T entity = it.next(); - if (!existingByFqn.containsKey(entity.getFullyQualifiedName())) { - it.remove(); - failedRequests.add( - new BulkResponse() - .withRequest(entity.getFullyQualifiedName()) - .withStatus(Status.BAD_REQUEST.getStatusCode()) - .withMessage("Entity does not exist and could not be created")); - } - } - } - - // Batch update existing entities - bulkUpdateEntities( + return bulkImportService.submitAsyncBulkOperation( uriInfo, - updateEntities, - existingByFqn, + entities, userName, + existingByFqn, overrideMetadata, - successRequests, - failedRequests, - entityLatenciesNanos); - - long totalDurationNanos = System.nanoTime() - bulkStartTime; - - result.setNumberOfRowsProcessed(entities.size()); - result.setNumberOfRowsPassed(successRequests.size()); - result.setNumberOfRowsFailed(failedRequests.size()); - result.setSuccessRequest(successRequests); - result.setFailedRequest(failedRequests); - - if (!failedRequests.isEmpty()) { - result.setStatus(successRequests.isEmpty() ? ApiStatus.FAILURE : ApiStatus.PARTIAL_SUCCESS); - } - - // Calculate metrics - long avgEntityLatencyMs = 0; - long maxEntityLatencyMs = 0; - if (!entityLatenciesNanos.isEmpty()) { - avgEntityLatencyMs = - entityLatenciesNanos.stream().mapToLong(Long::longValue).sum() - / entityLatenciesNanos.size() - / 1_000_000; - maxEntityLatencyMs = - entityLatenciesNanos.stream().mapToLong(Long::longValue).max().orElse(0) / 1_000_000; - } - - recordBulkMetrics( - entityType, - entities.size(), - successRequests.size(), - totalDurationNanos, - avgEntityLatencyMs, - maxEntityLatencyMs); - - LOG.info( - "Bulk operation completed: {} succeeded, {} failed out of {} total, took {}ms", - successRequests.size(), - failedRequests.size(), - entities.size(), - totalDurationNanos / 1_000_000); - - return result; - } - - public Optional getBulkJobStatus(String jobId) { - CompletableFuture job = BULK_JOBS.get(jobId); - if (job == null) { - return Optional.empty(); - } - - if (job.isDone() && !job.isCompletedExceptionally()) { - try { - return Optional.of(job.get()); - } catch (ExecutionException | InterruptedException e) { - LOG.error("Error retrieving job status for jobId: {}", jobId, e); - java.lang.Thread.currentThread().interrupt(); - return Optional.empty(); - } - } - - BulkOperationResult inProgress = new BulkOperationResult(); - inProgress.setStatus(ApiStatus.RUNNING); - return Optional.of(inProgress); - } - - @Transaction - private PutResponse createOrUpdateWithOriginal( - UriInfo uriInfo, T updated, T original, String updatedBy) { - if (lockManager != null) { - lockManager.checkModificationAllowed(updated); - } - if (original == null) { - return new PutResponse<>( - Status.CREATED, withHref(uriInfo, createNewEntity(updated)), ENTITY_CREATED); - } - return update(uriInfo, original, updated, updatedBy, null); - } - - private void createChangeEventForBulkOperation(T entity, EventType eventType, String userName) { - Optional changeEventJson = - buildChangeEventJsonForBulkOperation(entity, eventType, userName); - if (changeEventJson.isEmpty()) { - return; - } - try { - Entity.getCollectionDAO().changeEventDAO().insert(changeEventJson.get()); - } catch (Exception e) { - LOG.error("Failed to create change event for bulk operation", e); - } + authFailedResponses, + totalRequests); } private Optional buildChangeEventJsonForBulkOperation( T entity, EventType eventType, String userName) { - try { - if (eventType.equals(ENTITY_NO_CHANGE)) { - return Optional.empty(); - } - - ChangeEvent changeEvent = - FormatterUtil.createChangeEventForEntity(userName, eventType, entity); - - if (changeEvent.getEntity() != null) { - Object entityObject = changeEvent.getEntity(); - changeEvent = copyChangeEvent(changeEvent); - changeEvent.setEntity(JsonUtils.pojoToMaskedJson(entityObject)); - } - - LOG.debug( - "Recording change event for bulk operation {}:{}:{}:{}", - changeEvent.getTimestamp(), - changeEvent.getEntityId(), - changeEvent.getEventType(), - changeEvent.getEntityType()); - - return Optional.of(JsonUtils.pojoToJson(changeEvent)); - } catch (Exception e) { - LOG.error("Failed to create change event for bulk operation", e); - return Optional.empty(); - } + return bulkImportService.buildChangeEventJsonForBulkOperation(entity, eventType, userName); } private void insertChangeEventsBatch(List changeEvents) { - if (changeEvents == null || changeEvents.isEmpty()) { - return; - } - try { - Entity.getCollectionDAO().changeEventDAO().insertBatch(changeEvents); - } catch (Exception batchError) { - LOG.error("Failed to insert change events batch", batchError); - } - } - - private static ChangeEvent copyChangeEvent(ChangeEvent changeEvent) { - return new ChangeEvent() - .withId(changeEvent.getId()) - .withEventType(changeEvent.getEventType()) - .withEntityId(changeEvent.getEntityId()) - .withEntityType(changeEvent.getEntityType()) - .withUserName(changeEvent.getUserName()) - .withImpersonatedBy(changeEvent.getImpersonatedBy()) - .withTimestamp(changeEvent.getTimestamp()) - .withChangeDescription(changeEvent.getChangeDescription()) - .withCurrentVersion(changeEvent.getCurrentVersion()) - .withPreviousVersion(changeEvent.getPreviousVersion()) - .withEntityFullyQualifiedName(changeEvent.getEntityFullyQualifiedName()); + bulkImportService.insertChangeEventsBatch(changeEvents); } public BulkOperationResult bulkCreateOrUpdateEntities( UriInfo uriInfo, List entities, String userName, Map existingByFqn) { - return bulkCreateOrUpdateEntities(uriInfo, entities, userName, existingByFqn, false); + return bulkImportService.bulkCreateOrUpdateEntities(uriInfo, entities, userName, existingByFqn); } public BulkOperationResult bulkCreateOrUpdateEntities( @@ -12511,287 +9943,12 @@ public BulkOperationResult bulkCreateOrUpdateEntities( String userName, Map existingByFqn, boolean overrideMetadata) { - return bulkCreateOrUpdateEntitiesSequential( + return bulkImportService.bulkCreateOrUpdateEntities( uriInfo, entities, userName, existingByFqn, overrideMetadata); } - /** - * Deletes entities of this type within {@code request.scopeFqn} that the ingestion connector did - * not report in the current run. The connector sends the set of FQNs it saw ({@code - * request.seenFqns}); any live entity under the scope whose FQN is not in that set is considered - * stale. By default the deletion is soft; pass {@code hardDelete=true} on the request to - * hard-delete the stale entities. - * - *

FQNs are compared by hash so quoting or case differences between the connector-supplied and - * stored values never cause spurious deletes. An empty scope yields zero deletions - it is never - * interpreted as "everything is stale". Each delete runs in its own transaction so a single - * failure does not roll back the rest of the batch. - */ public BulkOperationResult bulkDeleteStaleEntities( BulkDeleteStaleRequest request, String deletedBy) { - validateScopeRequest(request.getScopeEntityType(), request.getScopeFqn()); - if (nullOrEmpty(request.getSeenFqns())) { - // An empty seen-set cannot be distinguished from a connector run that crashed or discovered - // nothing, so it must never be interpreted as "every entity under the scope is stale" - that - // would silently delete the whole service/database. Treat it as zero deletions, mirroring the - // scope-not-found path. - LOG.warn( - "deleteStale for scope {} '{}' received an empty seenFqns; treating as zero deletions " - + "rather than marking the entire scope stale", - request.getScopeEntityType(), - request.getScopeFqn()); - return buildStaleDeletionResult( - Boolean.TRUE.equals(request.getDryRun()), new ArrayList<>(), new ArrayList<>()); - } - if (!scopeExists(request.getScopeEntityType(), request.getScopeFqn())) { - LOG.warn( - "deleteStale scope {} '{}' not found; nothing to delete this run", - request.getScopeEntityType(), - request.getScopeFqn()); - return buildStaleDeletionResult( - Boolean.TRUE.equals(request.getDryRun()), new ArrayList<>(), new ArrayList<>()); - } - boolean dryRun = Boolean.TRUE.equals(request.getDryRun()); - boolean hardDelete = Boolean.TRUE.equals(request.getHardDelete()); - boolean recursive = !Boolean.FALSE.equals(request.getRecursive()); - List successRequests = new ArrayList<>(); - List failedRequests = new ArrayList<>(); - Set deletedHashes = new HashSet<>(); - for (EntityDAO.EntityIdFqnPair stale : - findStaleEntities(request.getScopeFqn(), request.getSeenFqns())) { - String fqnHash = FullyQualifiedName.buildHash(stale.fqn); - if (dryRun || isCoveredByDeletedAncestor(fqnHash, deletedHashes)) { - successRequests.add(staleSuccess(stale.fqn)); - continue; - } - try { - deleteInternal(deletedBy, stale.id, recursive, hardDelete); - deletedHashes.add(fqnHash); - successRequests.add(staleSuccess(stale.fqn)); - } catch (Exception e) { - LOG.warn("Failed to delete stale {} '{}': {}", entityType, stale.fqn, e.getMessage()); - failedRequests.add( - new BulkResponse() - .withRequest(stale.fqn) - .withStatus(Status.INTERNAL_SERVER_ERROR.getStatusCode()) - .withMessage(e.getMessage())); - } - } - return buildStaleDeletionResult(dryRun, successRequests, failedRequests); - } - - /** - * Rejects a malformed request before any work runs: {@code scopeFqn} and {@code scopeEntityType} - * must be present and {@code scopeEntityType} must resolve to a registered entity type. These are - * caller mistakes (typos, swapped fields) and fail with 400. An empty {@code seenFqns} is handled - * separately in {@link #bulkDeleteStaleEntities} as a safe zero-deletion no-op rather than a 400, - * since it is a well-formed request whose intent is genuinely ambiguous. - */ - private void validateScopeRequest(String scopeEntityType, String scopeFqn) { - if (nullOrEmpty(scopeFqn)) { - throw BadRequestException.of("scopeFqn is required for deleteStale"); - } - if (nullOrEmpty(scopeEntityType)) { - throw BadRequestException.of("scopeEntityType is required for deleteStale"); - } - if (!Entity.hasEntityRepository(resolveScopeEntityType(scopeEntityType))) { - throw BadRequestException.of( - String.format("Unsupported scopeEntityType '%s' for deleteStale", scopeEntityType)); - } - } - - /** - * Returns whether a live entity with FQN {@code scopeFqn} exists under the resolved scope type. A - * missing scope is not an error - it means the connector has nothing to reconcile yet (scope not - * persisted) or the scope was already removed - so the caller treats it as zero deletions. - */ - private boolean scopeExists(String scopeEntityType, String scopeFqn) { - EntityRepository scopeRepository = - Entity.getEntityRepository(resolveScopeEntityType(scopeEntityType)); - return scopeRepository.findByNameOrNull(scopeFqn, Include.NON_DELETED) != null; - } - - /** - * Resolves a generic {@code service} scope to the concrete service type that owns this entity - * type (for example {@code pipeline} -> {@code pipelineService}), mirroring how {@link - * org.openmetadata.service.jdbi3.ListFilter} interprets the {@code service} query param. - * Connectors scope service-level stale deletion with the generic {@code service} key, so without - * this the request would be rejected as an unknown entity type. Any concrete scope type is - * returned unchanged. - */ - private String resolveScopeEntityType(String scopeEntityType) { - return Entity.FIELD_SERVICE.equals(scopeEntityType) - ? Entity.getServiceType(entityType) - : scopeEntityType; - } - - /** - * Returns the live entities under {@code scopeFqn} that are not present in {@code seenFqns}, - * sorted shallowest-FQN-first so a recursive delete of an ancestor is processed before its - * descendants. Comparison is by FQN hash to be quoting and case insensitive. - */ - private List findStaleEntities( - String scopeFqn, List seenFqns) { - List scopeEntities = - dao.listDescendantIdFqnByPrefixNonDeleted(scopeFqn); - if (scopeEntities.isEmpty()) { - return List.of(); - } - Set seenHashes = new HashSet<>(); - for (String seenFqn : listOrEmpty(seenFqns)) { - try { - seenHashes.add(FullyQualifiedName.buildHash(seenFqn)); - } catch (Exception e) { - LOG.warn("Ignoring malformed seen FQN '{}' in stale deletion request", seenFqn); - } - } - return scopeEntities.stream() - .filter(pair -> !seenHashes.contains(FullyQualifiedName.buildHash(pair.fqn))) - .sorted(Comparator.comparingInt(pair -> FullyQualifiedName.split(pair.fqn).length)) - .toList(); - } - - private boolean isCoveredByDeletedAncestor(String fqnHash, Set deletedHashes) { - if (deletedHashes.isEmpty()) { - return false; - } - int separator = fqnHash.lastIndexOf('.'); - while (separator > 0) { - String ancestor = fqnHash.substring(0, separator); - if (deletedHashes.contains(ancestor)) { - return true; - } - separator = ancestor.lastIndexOf('.'); - } - return false; - } - - private BulkResponse staleSuccess(String fqn) { - return new BulkResponse().withRequest(fqn).withStatus(Status.OK.getStatusCode()); - } - - private BulkOperationResult buildStaleDeletionResult( - boolean dryRun, List successRequests, List failedRequests) { - BulkOperationResult result = new BulkOperationResult(); - result.setDryRun(dryRun); - result.setStatus(failedRequests.isEmpty() ? ApiStatus.SUCCESS : ApiStatus.PARTIAL_SUCCESS); - result.setNumberOfRowsProcessed(successRequests.size() + failedRequests.size()); - result.setNumberOfRowsPassed(successRequests.size()); - result.setNumberOfRowsFailed(failedRequests.size()); - result.setSuccessRequest(successRequests); - result.setFailedRequest(failedRequests); - return result; - } - - private static boolean isDuplicateKeyException(Exception e) { - Throwable cause = e.getCause(); - if (cause instanceof SQLException sqlEx) { - // MySQL: error code 1062 = ER_DUP_ENTRY - // PostgreSQL: SQL state "23505" = unique_violation - return sqlEx.getErrorCode() == 1062 || "23505".equals(sqlEx.getSQLState()); - } - return false; - } - - private void recordEntityMetrics( - String entityType, long durationNanos, long queueWaitNanos, boolean success) { - // Per-entity processing time (cached, no histogram to reduce Prometheus cardinality) - // This fires for EVERY entity in a bulk operation, so we use simple timers. - // The bulk.operation.latency metric has histograms for percentile analysis. - String latencyKey = entityType + "|" + success; - Timer latencyTimer = - ENTITY_LATENCY_TIMERS.computeIfAbsent( - latencyKey, - k -> - Timer.builder("bulk.entity.latency") - .tag("entity", entityType) - .tag("success", String.valueOf(success)) - .register(Metrics.globalRegistry)); - latencyTimer.record(durationNanos, TimeUnit.NANOSECONDS); - - // Queue wait time (cached, simple timer) - Timer queueTimer = - ENTITY_QUEUE_WAIT_TIMERS.computeIfAbsent( - entityType, - k -> - Timer.builder("bulk.entity.queue_wait") - .tag("entity", entityType) - .register(Metrics.globalRegistry)); - queueTimer.record(queueWaitNanos, TimeUnit.NANOSECONDS); - } - - private void recordBulkMetrics( - String entityType, - int totalEntities, - int successCount, - long durationNanos, - long avgEntityMs, - long maxEntityMs) { - // Total bulk operation time (cached) - Timer operationTimer = - BULK_OPERATION_TIMERS.computeIfAbsent( - entityType, - k -> - Timer.builder("bulk.operation.latency") - .tag("entity", entityType) - .publishPercentileHistogram(true) - .register(Metrics.globalRegistry)); - operationTimer.record(durationNanos, TimeUnit.NANOSECONDS); - - // Batch size distribution (cached) - DistributionSummary batchSizeSummary = - BATCH_SIZE_SUMMARIES.computeIfAbsent( - entityType, - k -> - DistributionSummary.builder("bulk.operation.batch_size") - .tag("entity", entityType) - .register(Metrics.globalRegistry)); - batchSizeSummary.record(totalEntities); - - // Success rate as distribution (cached, avoids gauge memory leak) - if (totalEntities > 0) { - DistributionSummary successRateSummary = - SUCCESS_RATE_SUMMARIES.computeIfAbsent( - entityType, - k -> - DistributionSummary.builder("bulk.operation.success_rate") - .tag("entity", entityType) - .register(Metrics.globalRegistry)); - successRateSummary.record(successCount * 100.0 / totalEntities); - } - - // Record success and failure counts for alerting (Micrometer caches counters internally) - Metrics.counter("bulk.operation.entities.success", "entity", entityType) - .increment(successCount); - Metrics.counter("bulk.operation.entities.failed", "entity", entityType) - .increment(totalEntities - successCount); - } - - private void handleBulkOperationError(T entity, Exception e, List failedRequests) { - String fqn = entity.getFullyQualifiedName(); - int statusCode; - String message; - - // Categorize errors properly - if (e instanceof jakarta.ws.rs.WebApplicationException wae) { - statusCode = wae.getResponse().getStatus(); - message = e.getMessage(); - LOG.warn("Entity {} failed with status {}: {}", fqn, statusCode, message); - } else if (e instanceof java.sql.SQLException) { - statusCode = Status.INTERNAL_SERVER_ERROR.getStatusCode(); - message = "Database error: " + e.getMessage(); - LOG.error("Database error processing entity {}", fqn, e); - } else if (e instanceof IllegalArgumentException || e instanceof IllegalStateException) { - statusCode = Status.BAD_REQUEST.getStatusCode(); - message = e.getMessage(); - LOG.warn("Validation error for entity {}: {}", fqn, message); - } else { - statusCode = Status.INTERNAL_SERVER_ERROR.getStatusCode(); - message = "Unexpected error: " + e.getMessage(); - LOG.error("Unexpected error processing entity {}", fqn, e); - } - - failedRequests.add( - new BulkResponse().withRequest(fqn).withStatus(statusCode).withMessage(message)); + return bulkImportService.bulkDeleteStaleEntities(request, deletedBy); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTaskWorkflows.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTaskWorkflows.java new file mode 100644 index 000000000000..0534a3e0c5df --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTaskWorkflows.java @@ -0,0 +1,108 @@ +/* + * Copyright 2024 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.Entity.FIELD_ENTITY_STATUS; +import static org.openmetadata.service.governance.workflows.Workflow.RESULT_VARIABLE; +import static org.openmetadata.service.governance.workflows.Workflow.UPDATED_BY_VARIABLE; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.feed.ResolveTask; +import org.openmetadata.schema.type.TagLabel; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.governance.workflows.WorkflowHandler; +import org.openmetadata.service.jdbi3.FeedRepository.TaskWorkflow; +import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext; +import org.openmetadata.service.util.DescriptionSanitizer; +import org.openmetadata.service.util.EntityFieldUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Generic task-resolution workflows for description, tag, and approval tasks. Extracted from + * EntityRepository as part of decomposing that class; the default {@code getTaskWorkflow} factory + * still instantiates these. + */ +class DescriptionTaskWorkflow extends TaskWorkflow { + DescriptionTaskWorkflow(ThreadContext threadContext) { + super(threadContext); + } + + @Override + public EntityInterface performTask(String user, ResolveTask resolveTask) { + EntityInterface aboutEntity = threadContext.getAboutEntity(); + aboutEntity.setDescription(DescriptionSanitizer.sanitize(resolveTask.getNewValue())); + return aboutEntity; + } +} + +class TagTaskWorkflow extends TaskWorkflow { + TagTaskWorkflow(ThreadContext threadContext) { + super(threadContext); + } + + @Override + public EntityInterface performTask(String user, ResolveTask resolveTask) { + List tags = JsonUtils.readObjects(resolveTask.getNewValue(), TagLabel.class); + EntityInterface aboutEntity = threadContext.getAboutEntity(); + aboutEntity.setTags(tags); + return aboutEntity; + } +} + +/** + * Generic approval task workflow usable for any entity. Checks that the acting user is a reviewer of + * the entity, then delegates resolution to the governance WorkflowHandler. Falls back to a direct + * entityStatus patch when the Flowable workflow record no longer exists. + */ +class ApprovalTaskWorkflow extends TaskWorkflow { + private static final Logger LOG = LoggerFactory.getLogger(ApprovalTaskWorkflow.class); + + ApprovalTaskWorkflow(ThreadContext threadContext) { + super(threadContext); + } + + @Override + public EntityInterface performTask(String user, ResolveTask resolveTask) { + EntityInterface entity = threadContext.getAboutEntity(); + EntityRepository.verifyReviewer(entity, user); + + UUID taskId = threadContext.getThread().getId(); + Map variables = new HashMap<>(); + variables.put(RESULT_VARIABLE, resolveTask.getNewValue().equalsIgnoreCase("approved")); + variables.put(UPDATED_BY_VARIABLE, user); + WorkflowHandler workflowHandler = WorkflowHandler.getInstance(); + boolean workflowSuccess = + workflowHandler.resolveLegacyThreadTask( + taskId, workflowHandler.transformToNodeVariables(taskId, variables)); + + if (!workflowSuccess) { + LOG.warn("Workflow failed for taskId='{}', applying status directly", taskId); + Boolean approved = (Boolean) variables.get(RESULT_VARIABLE); + String entityStatus = (approved != null && approved) ? "Approved" : "Rejected"; + EntityFieldUtils.setEntityField( + entity, + threadContext.getAbout().getEntityType(), + user, + FIELD_ENTITY_STATUS, + entityStatus, + true); + } + + return entity; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionDAOs.java new file mode 100644 index 000000000000..2d96223aae24 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EventSubscriptionDAOs.java @@ -0,0 +1,271 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.util.List; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.schema.entity.events.EventSubscription; +import org.openmetadata.schema.entity.events.NotificationTemplate; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlBatch; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; + +public interface EventSubscriptionDAOs { + @CreateSqlObject + EventSubscriptionDAO eventSubscriptionDAO(); + + @CreateSqlObject + NotificationTemplateDAO notificationTemplateDAO(); + + interface EventSubscriptionDAO extends EntityDAO { + @Override + default String getTableName() { + return "event_subscription_entity"; + } + + @Override + default Class getEntityClass() { + return EventSubscription.class; + } + + @SqlQuery("SELECT json FROM event_subscription_entity") + List listAllEventsSubscriptions(); + + @Override + default boolean supportsSoftDelete() { + return false; + } + + @SqlQuery("SELECT json FROM change_event_consumers where id = :id AND extension = :extension") + String getSubscriberExtension(@Bind("id") String id, @Bind("extension") String extension); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO change_event_consumers(id, extension, jsonSchema, json) " + + "VALUES (:id, :extension, :jsonSchema, :json)" + + "ON DUPLICATE KEY UPDATE json = :json, jsonSchema = :jsonSchema", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO change_event_consumers(id, extension, jsonSchema, json) " + + "VALUES (:id, :extension, :jsonSchema, (:json :: jsonb)) ON CONFLICT (id, extension) " + + "DO UPDATE SET json = EXCLUDED.json, jsonSchema = EXCLUDED.jsonSchema", + connectionType = POSTGRES) + void upsertSubscriberExtension( + @Bind("id") String id, + @Bind("extension") String extension, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO consumers_dlq(id, extension, json, source) " + + "VALUES (:id, :extension, :json, :source) " + + "ON DUPLICATE KEY UPDATE json = :json, source = :source", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO consumers_dlq(id, extension, json, source) " + + "VALUES (:id, :extension, (:json :: jsonb), :source) " + + "ON CONFLICT (id, extension) " + + "DO UPDATE SET json = EXCLUDED.json, source = EXCLUDED.source", + connectionType = POSTGRES) + void upsertFailedEvent( + @Bind("id") String id, + @Bind("extension") String extension, + @Bind("json") String json, + @Bind("source") String source); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO successful_sent_change_events (change_event_id, event_subscription_id, json, timestamp) " + + "VALUES (:change_event_id, :event_subscription_id, :json, :timestamp) " + + "ON DUPLICATE KEY UPDATE json = :json, timestamp = :timestamp", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO successful_sent_change_events (change_event_id, event_subscription_id, json, timestamp) " + + "VALUES (:change_event_id, :event_subscription_id, CAST(:json AS jsonb), :timestamp) " + + "ON CONFLICT (change_event_id, event_subscription_id) " + + "DO UPDATE SET json = EXCLUDED.json, timestamp = EXCLUDED.timestamp", + connectionType = POSTGRES) + void upsertSuccessfulChangeEvent( + @Bind("change_event_id") String changeEventId, + @Bind("event_subscription_id") String eventSubscriptionId, + @Bind("json") String json, + @Bind("timestamp") long timestamp); + + // Batch insert for successful events - reduces connection pool contention + // from N connections to 1 when processing multiple events + @Transaction + @ConnectionAwareSqlBatch( + value = + "INSERT INTO successful_sent_change_events (change_event_id, event_subscription_id, json, timestamp) " + + "VALUES (:change_event_id, :event_subscription_id, :json, :timestamp) " + + "ON DUPLICATE KEY UPDATE json = VALUES(json), timestamp = VALUES(timestamp)", + connectionType = MYSQL) + @ConnectionAwareSqlBatch( + value = + "INSERT INTO successful_sent_change_events (change_event_id, event_subscription_id, json, timestamp) " + + "VALUES (:change_event_id, :event_subscription_id, CAST(:json AS jsonb), :timestamp) " + + "ON CONFLICT (change_event_id, event_subscription_id) " + + "DO UPDATE SET json = EXCLUDED.json, timestamp = EXCLUDED.timestamp", + connectionType = POSTGRES) + void batchUpsertSuccessfulChangeEvents( + @Bind("change_event_id") List changeEventIds, + @Bind("event_subscription_id") List eventSubscriptionIds, + @Bind("json") List jsonList, + @Bind("timestamp") List timestamps); + + @SqlQuery( + "SELECT COUNT(*) FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId") + long getSuccessfulRecordCount(@Bind("eventSubscriptionId") String eventSubscriptionId); + + @SqlQuery( + "SELECT event_subscription_id FROM successful_sent_change_events " + + "GROUP BY event_subscription_id " + + "HAVING COUNT(*) >= :threshold") + List findSubscriptionsAboveThreshold(@Bind("threshold") int threshold); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId ORDER BY timestamp ASC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM successful_sent_change_events WHERE ctid IN (SELECT ctid FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId ORDER BY timestamp ASC LIMIT :limit)", + connectionType = POSTGRES) + void deleteOldRecords( + @Bind("eventSubscriptionId") String eventSubscriptionId, @Bind("limit") long limit); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM successful_sent_change_events " + + "WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM successful_sent_change_events " + + "WHERE ctid IN ( " + + " SELECT ctid FROM successful_sent_change_events " + + " WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit " + + ")", + connectionType = POSTGRES) + int deleteSuccessfulSentChangeEventsInBatches( + @Bind("cutoff") long cutoff, @Bind("limit") int limit); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM change_event " + + "WHERE eventTime < :cutoff ORDER BY eventTime LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM change_event " + + "WHERE ctid IN ( " + + " SELECT ctid FROM change_event " + + " WHERE eventTime < :cutoff ORDER BY eventTime LIMIT :limit " + + ")", + connectionType = POSTGRES) + int deleteChangeEventsInBatches(@Bind("cutoff") long cutoff, @Bind("limit") int limit); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM consumers_dlq " + + "WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM consumers_dlq " + + "WHERE ctid IN ( " + + " SELECT ctid FROM consumers_dlq " + + " WHERE timestamp < :cutoff ORDER BY timestamp LIMIT :limit " + + ")", + connectionType = POSTGRES) + int deleteConsumersDlqInBatches(@Bind("cutoff") long cutoff, @Bind("limit") int limit); + + @SqlQuery( + "SELECT json FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId ORDER BY timestamp DESC LIMIT :limit OFFSET :paginationOffset") + List getSuccessfulChangeEventBySubscriptionId( + @Bind("eventSubscriptionId") String eventSubscriptionId, + @Bind("limit") int limit, + @Bind("paginationOffset") long paginationOffset); + + @SqlUpdate( + "DELETE FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId") + void deleteSuccessfulChangeEventBySubscriptionId( + @Bind("eventSubscriptionId") String eventSubscriptionId); + + @SqlUpdate("DELETE FROM consumers_dlq WHERE id = :eventSubscriptionId") + void deleteFailedRecordsBySubscriptionId( + @Bind("eventSubscriptionId") String eventSubscriptionId); + + @SqlUpdate("DELETE from change_event_consumers cec where id = :eventSubscriptionId;") + void deleteAlertMetrics(@Bind("eventSubscriptionId") String eventSubscriptionId); + + @ConnectionAwareSqlQuery( + value = + "SELECT COUNT(*) FROM ( " + + " SELECT json, 'FAILED' AS status, timestamp " + + " FROM consumers_dlq WHERE id = :id " + + " UNION ALL " + + " SELECT json, 'SUCCESSFUL' AS status, timestamp " + + " FROM successful_sent_change_events WHERE event_subscription_id = :id " + + ") AS combined_events", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT COUNT(*) FROM ( " + + " SELECT json, 'failed' AS status, timestamp " + + " FROM consumers_dlq WHERE id = :id " + + " UNION ALL " + + " SELECT json, 'successful' AS status, timestamp " + + " FROM successful_sent_change_events WHERE event_subscription_id = :id " + + ") AS combined_events", + connectionType = POSTGRES) + int countAllEventsWithStatuses(@Bind("id") String id); + + @SqlQuery("SELECT COUNT(*) FROM consumers_dlq WHERE id = :id") + int countFailedEventsById(@Bind("id") String id); + + @SqlQuery( + "SELECT COUNT(*) FROM successful_sent_change_events WHERE event_subscription_id = :eventSubscriptionId") + int countSuccessfulEventsBySubscriptionId( + @Bind("eventSubscriptionId") String eventSubscriptionId); + } + + interface NotificationTemplateDAO extends EntityDAO { + @Override + default String getTableName() { + return "notification_template_entity"; + } + + @Override + default Class getEntityClass() { + return NotificationTemplate.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedDAOs.java new file mode 100644 index 000000000000..3c0ad6e19970 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedDAOs.java @@ -0,0 +1,2014 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.schema.type.Relationship.MENTIONED_IN; +import static org.openmetadata.service.Entity.GLOSSARY_TERM; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.customizer.BindMap; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.openmetadata.schema.entity.feed.Announcement; +import org.openmetadata.schema.entity.feed.TaskFormSchema; +import org.openmetadata.schema.entity.tasks.Task; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.jdbi3.FeedRepository.FilterType; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; +import org.openmetadata.service.util.jdbi.BindConcat; +import org.openmetadata.service.util.jdbi.BindFQN; +import org.openmetadata.service.util.jdbi.BindUUID; + +public interface FeedDAOs { + @CreateSqlObject + FeedDAO feedDAO(); + + @CreateSqlObject + TaskDAO taskDAO(); + + @CreateSqlObject + AnnouncementDAO announcementDAO(); + + @CreateSqlObject + TaskFormSchemaDAO taskFormSchemaDAO(); + + interface FeedDAO { + @ConnectionAwareSqlUpdate( + value = "INSERT INTO (json) VALUES (:json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO (json) VALUES (:json :: jsonb)", + connectionType = POSTGRES) + void insert(@Define("tableName") String tableName, @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "INSERT INTO thread_entity(json) VALUES (:json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO thread_entity(json) VALUES (:json :: jsonb)", + connectionType = POSTGRES) + void insert(@Bind("json") String json); + + @SqlQuery("SELECT json FROM WHERE id = :id") + String findById(@Define("tableName") String tableName, @BindUUID("id") UUID id); + + @SqlQuery("SELECT json FROM thread_entity WHERE id = :id") + String findById(@BindUUID("id") UUID id); + + @SqlQuery("SELECT json FROM ORDER BY createdAt DESC") + List list(@Define("tableName") String tableName); + + @SqlQuery("SELECT json FROM thread_entity ORDER BY createdAt DESC") + List list(); + + @SqlQuery("SELECT count(id) FROM ") + int listCount( + @Define("tableName") String tableName, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery("SELECT count(id) FROM thread_entity ") + int listCount(@Define("condition") String condition, @BindMap Map params); + + @SqlUpdate("DELETE FROM WHERE id = :id") + void delete(@Define("tableName") String tableName, @BindUUID("id") UUID id); + + @SqlUpdate("DELETE FROM thread_entity WHERE id = :id") + void delete(@BindUUID("id") UUID id); + + @SqlUpdate("DELETE FROM WHERE id IN ()") + int deleteByIds(@Define("tableName") String tableName, @BindList("ids") List ids); + + @SqlUpdate("DELETE FROM thread_entity WHERE id IN ()") + int deleteByIds(@BindList("ids") List ids); + + @ConnectionAwareSqlUpdate( + value = "UPDATE task_sequence SET id=LAST_INSERT_ID(id+1)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE task_sequence SET id=(id+1) RETURNING id", + connectionType = POSTGRES) + void updateTaskId(); + + @ConnectionAwareSqlQuery(value = "SELECT LAST_INSERT_ID()", connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT id FROM task_sequence LIMIT 1", + connectionType = POSTGRES) + int getTaskId(); + + @SqlQuery("SELECT json FROM WHERE taskId = :id") + String findByTaskId(@Define("tableName") String tableName, @Bind("id") int id); + + @SqlQuery("SELECT json FROM thread_entity WHERE taskId = :id") + String findByTaskId(@Bind("id") int id); + + @SqlQuery("SELECT json FROM ORDER BY createdAt DESC LIMIT :limit") + List list( + @Define("tableName") String tableName, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery("SELECT json FROM thread_entity ORDER BY createdAt DESC LIMIT :limit") + List list( + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM " + + "WHERE type='Announcement' AND (:threadId IS NULL OR id != :threadId) " + + "AND entityId = :entityId " + + "AND (( :startTs >= announcementStart AND :startTs < announcementEnd) " + + "OR (:endTs > announcementStart AND :endTs < announcementEnd) " + + "OR (:startTs <= announcementStart AND :endTs >= announcementEnd))") + List listAnnouncementBetween( + @Define("tableName") String tableName, + @BindUUID("threadId") UUID threadId, + @BindUUID("entityId") UUID entityId, + @Bind("startTs") long startTs, + @Bind("endTs") long endTs); + + @SqlQuery( + "SELECT json FROM thread_entity " + + "WHERE type='Announcement' AND (:threadId IS NULL OR id != :threadId) " + + "AND entityId = :entityId " + + "AND (( :startTs >= announcementStart AND :startTs < announcementEnd) " + + "OR (:endTs > announcementStart AND :endTs < announcementEnd) " + + "OR (:startTs <= announcementStart AND :endTs >= announcementEnd))") + List listAnnouncementBetween( + @BindUUID("threadId") UUID threadId, + @BindUUID("entityId") UUID entityId, + @Bind("startTs") long startTs, + @Bind("endTs") long endTs); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM AND " + + "to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit", + connectionType = POSTGRES) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM AND " + + "MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit", + connectionType = MYSQL) + List listTasksAssigned( + @Define("tableName") String tableName, + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM thread_entity AND " + + "to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit", + connectionType = POSTGRES) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM thread_entity AND " + + "MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit", + connectionType = MYSQL) + List listTasksAssigned( + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM AND " + + "to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) ", + connectionType = POSTGRES) + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM AND " + + "MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) ", + connectionType = MYSQL) + int listCountTasksAssignedTo( + @Define("tableName") String tableName, + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Define("condition") String condition, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM thread_entity AND " + + "to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) ", + connectionType = POSTGRES) + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM thread_entity AND " + + "MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) ", + connectionType = MYSQL) + int listCountTasksAssignedTo( + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Define("condition") String condition, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM " + + "AND (to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) OR createdBy = :username) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit", + connectionType = POSTGRES) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM " + + "AND (MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) OR createdBy = :username) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit", + connectionType = MYSQL) + List listTasksOfUser( + @Define("tableName") String tableName, + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Bind("username") String username, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM thread_entity " + + "AND (to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) OR createdBy = :username) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit", + connectionType = POSTGRES) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM thread_entity " + + "AND (MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) OR createdBy = :username) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit", + connectionType = MYSQL) + List listTasksOfUser( + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Bind("username") String username, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT id FROM WHERE type = 'Conversation' AND createdAt < :cutoffMillis LIMIT :batchSize") + List fetchConversationThreadIdsOlderThan( + @Define("tableName") String tableName, + @Bind("cutoffMillis") long cutoffMillis, + @Bind("batchSize") int batchSize); + + @SqlQuery( + "SELECT id FROM thread_entity WHERE type = 'Conversation' AND createdAt < :cutoffMillis LIMIT :batchSize") + List fetchConversationThreadIdsOlderThan( + @Bind("cutoffMillis") long cutoffMillis, @Bind("batchSize") int batchSize); + + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM " + + "AND (to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) OR createdBy = :username) ", + connectionType = POSTGRES) + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM " + + "AND (MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) OR createdBy = :username) ", + connectionType = MYSQL) + int listCountTasksOfUser( + @Define("tableName") String tableName, + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Bind("username") String username, + @Define("condition") String condition, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM thread_entity " + + "AND (to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) OR createdBy = :username) ", + connectionType = POSTGRES) + @ConnectionAwareSqlQuery( + value = + "SELECT count(id) FROM thread_entity " + + "AND (MATCH(taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) OR createdBy = :username) ", + connectionType = MYSQL) + int listCountTasksOfUser( + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Bind("username") String username, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM AND createdBy = :username ORDER BY createdAt DESC LIMIT :limit") + List listTasksAssignedByUser( + @Define("tableName") String tableName, + @Bind("username") String username, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM thread_entity AND createdBy = :username ORDER BY createdAt DESC LIMIT :limit") + List listTasksAssigned( + @Bind("username") String username, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery("SELECT count(id) FROM AND createdBy = :username") + int listCountTasksAssignedBy( + @Define("tableName") String tableName, + @Bind("username") String username, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery("SELECT count(id) FROM thread_entity AND createdBy = :username") + int listCountTasksAssignedBy( + @Bind("username") String username, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM thread_entity where type = 'Task' LIMIT :limit OFFSET :paginationOffset") + List listTaskThreadWithOffset( + @Bind("limit") int limit, @Bind("paginationOffset") int paginationOffset); + + @SqlQuery( + "SELECT json FROM thread_entity where type != 'Task' AND createdAt > :cutoffMillis ORDER BY createdAt LIMIT :limit OFFSET :paginationOffset") + List listOtherConversationThreadWithOffset( + @Bind("cutoffMillis") long cutoffMillis, + @Bind("limit") int limit, + @Bind("paginationOffset") int paginationOffset); + + @SqlQuery( + "SELECT json FROM AND " + // Entity for which the thread is about is owned by the user or his teams + + "(entityId in (SELECT toId FROM entity_relationship WHERE " + + "((fromEntity='user' AND fromId= :userId) OR " + + "(fromEntity='team' AND fromId IN ())) AND relation=8) OR " + + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2)))) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByOwner( + @Define("tableName") String tableName, + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM thread_entity AND " + // Entity for which the thread is about is owned by the user or his teams + + "(entityId in (SELECT toId FROM entity_relationship WHERE " + + "((fromEntity='user' AND fromId= :userId) OR " + + "(fromEntity='team' AND fromId IN ())) AND relation=8) OR " + + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2)))) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByOwner( + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT count(id) FROM AND " + + "(entityId in (SELECT toId FROM entity_relationship WHERE " + + "((fromEntity='user' AND fromId= :userId) OR " + + "(fromEntity='team' AND fromId IN ())) AND relation=8) OR " + + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2)))) ") + int listCountThreadsByOwner( + @Define("tableName") String tableName, + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT count(id) FROM thread_entity AND " + + "(entityId in (SELECT toId FROM entity_relationship WHERE " + + "((fromEntity='user' AND fromId= :userId) OR " + + "(fromEntity='team' AND fromId IN ())) AND relation=8) OR " + + "id in (SELECT toId FROM entity_relationship WHERE (fromEntity='user' AND fromId= :userId AND toEntity='THREAD' AND relation IN (1,2)))) ") + int listCountThreadsByOwner( + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + value = + "SELECT json " + + " FROM " + + " WHERE testCaseResolutionStatusId = :testCaseResolutionStatusId") + String fetchThreadByTestCaseResolutionStatusId( + @Define("tableName") String tableName, + @BindUUID("testCaseResolutionStatusId") UUID testCaseResolutionStatusId); + + @SqlQuery( + value = + "SELECT json " + + " FROM thread_entity " + + " WHERE testCaseResolutionStatusId = :testCaseResolutionStatusId") + String fetchThreadByTestCaseResolutionStatusId( + @BindUUID("testCaseResolutionStatusId") UUID testCaseResolutionStatusId); + + default List listThreadsByEntityLink( + String tableName, + FeedFilter filter, + EntityLink entityLink, + int limit, + int relation, + String userName, + List teamNames) { + int filterRelation = -1; + if (userName != null && filter.getFilterType() == FilterType.MENTIONS) { + filterRelation = MENTIONED_IN.ordinal(); + } + return listThreadsByEntityLink( + tableName, + entityLink.getFullyQualifiedFieldValue(), + entityLink.getFullyQualifiedFieldType(), + limit, + relation, + userName, + teamNames, + filterRelation, + filter.getCondition(), + filter.getQueryParams()); + } + + default List listThreadsByEntityLink( + FeedFilter filter, + EntityLink entityLink, + int limit, + int relation, + String userName, + List teamNames) { + int filterRelation = -1; + if (userName != null && filter.getFilterType() == FilterType.MENTIONS) { + filterRelation = MENTIONED_IN.ordinal(); + } + return listThreadsByEntityLink( + entityLink.getFullyQualifiedFieldValue(), + entityLink.getFullyQualifiedFieldType(), + limit, + relation, + userName, + teamNames, + filterRelation, + filter.getCondition(), + filter.getQueryParams()); + } + + @SqlQuery( + "SELECT json FROM " + + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " + + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " + + " ((fromType='user' AND fromFQNHash= :userName) OR" + + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByEntityLink( + @Define("tableName") String tableName, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType, + @Bind("limit") int limit, + @Bind("relation") int relation, + @BindFQN("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("filterRelation") int filterRelation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM thread_entity " + + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " + + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " + + " ((fromType='user' AND fromFQNHash= :userName) OR" + + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByEntityLink( + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType, + @Bind("limit") int limit, + @Bind("relation") int relation, + @BindFQN("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("filterRelation") int filterRelation, + @Define("condition") String condition, + @BindMap Map params); + + default int listCountThreadsByEntityLink( + String tableName, + FeedFilter filter, + EntityLink entityLink, + int relation, + String userName, + List teamNames) { + int filterRelation = -1; + if (userName != null && filter.getFilterType() == FilterType.MENTIONS) { + filterRelation = MENTIONED_IN.ordinal(); + } + return listCountThreadsByEntityLink( + tableName, + entityLink.getFullyQualifiedFieldValue(), + entityLink.getFullyQualifiedFieldType(), + relation, + userName, + teamNames, + filterRelation, + filter.getCondition(false), + filter.getQueryParams()); + } + + default int listCountThreadsByEntityLink( + FeedFilter filter, + EntityLink entityLink, + int relation, + String userName, + List teamNames) { + int filterRelation = -1; + if (userName != null && filter.getFilterType() == FilterType.MENTIONS) { + filterRelation = MENTIONED_IN.ordinal(); + } + return listCountThreadsByEntityLink( + entityLink.getFullyQualifiedFieldValue(), + entityLink.getFullyQualifiedFieldType(), + relation, + userName, + teamNames, + filterRelation, + filter.getCondition(false), + filter.getQueryParams()); + } + + @SqlQuery( + "SELECT count(id) FROM " + + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " + + "AND (:userName IS NULL OR id in (SELECT toFQNHash FROM field_relationship WHERE " + + " ((fromType='user' AND fromFQNHash= :userName) OR" + + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )") + int listCountThreadsByEntityLink( + @Define("tableName") String tableName, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType, + @Bind("relation") int relation, + @Bind("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("filterRelation") int filterRelation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT count(id) FROM thread_entity " + + "AND hash_id in (SELECT fromFQNHash FROM field_relationship WHERE " + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "(:toType IS NULL OR toType LIKE :concatToType OR toType=:toType) AND relation= :relation) " + + "AND (:userName IS NULL OR id in (SELECT toFQNHash FROM field_relationship WHERE " + + " ((fromType='user' AND fromFQNHash= :userName) OR" + + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )") + int listCountThreadsByEntityLink( + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType, + @Bind("relation") int relation, + @Bind("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("filterRelation") int filterRelation, + @Define("condition") String condition, + @BindMap Map params); + + @ConnectionAwareSqlUpdate( + value = "UPDATE SET json = :json where id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE SET json = (:json :: jsonb) where id = :id", + connectionType = POSTGRES) + void update( + @Define("tableName") String tableName, @BindUUID("id") UUID id, @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "UPDATE thread_entity SET json = :json where id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE thread_entity SET json = (:json :: jsonb) where id = :id", + connectionType = POSTGRES) + void update(@BindUUID("id") UUID id, @Bind("json") String json); + + @SqlQuery( + "SELECT entityLink, type, taskStatus, COUNT(id) as count FROM ( " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE hash_id IN ( " + + " SELECT fromFQNHash FROM field_relationship " + + " WHERE " + + " (:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash = :fqnPrefixHash) " + + " AND fromType = 'THREAD' " + + " AND (:toType IS NULL OR toType LIKE :concatToType OR toType = :toType) " + + " AND relation = 3 " + + " ) " + + " UNION " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE te.entityId = :entityId " + + ") AS combined WHERE combined.type IS NOT NULL " + + "GROUP BY type, taskStatus, entityLink") + @RegisterRowMapper(ThreadCountFieldMapper.class) + List> listCountByEntityLink( + @Define("tableName") String tableName, + @BindUUID("entityId") UUID entityId, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType); + + @SqlQuery( + "SELECT entityLink, type, taskStatus, COUNT(id) as count FROM ( " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE hash_id IN ( " + + " SELECT fromFQNHash FROM field_relationship " + + " WHERE " + + " (:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash = :fqnPrefixHash) " + + " AND fromType = 'THREAD' " + + " AND (:toType IS NULL OR toType LIKE :concatToType OR toType = :toType) " + + " AND relation = 3 " + + " ) " + + " UNION " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE te.entityId = :entityId " + + ") AS combined WHERE combined.type IS NOT NULL " + + "GROUP BY type, taskStatus, entityLink") + @RegisterRowMapper(ThreadCountFieldMapper.class) + List> listCountByEntityLink( + @BindUUID("entityId") UUID entityId, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType", + original = "toType", + parts = {":toType", ".%"}) + String toType); + + @ConnectionAwareSqlQuery( + value = + "SELECT COUNT(te.id) AS count " + + "FROM te " + + "WHERE te.type = 'Announcement' " + + " AND te.entityLink = :entityLink " + + " AND CAST(JSON_EXTRACT(te.json, '$.announcement.startTime') AS UNSIGNED) <= UNIX_TIMESTAMP()*1000 " + + " AND CAST(JSON_EXTRACT(te.json, '$.announcement.endTime') AS UNSIGNED) >= UNIX_TIMESTAMP()*1000", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT COUNT(te.id) AS count " + + "FROM te " + + "WHERE te.type = 'Announcement' " + + " AND te.entityLink = :entityLink " + + " AND (te.json->'announcement'->>'startTime')::numeric <= EXTRACT(EPOCH FROM NOW()) * 1000 " + + " AND (te.json->'announcement'->>'endTime')::numeric >= EXTRACT(EPOCH FROM NOW()) * 1000", + connectionType = POSTGRES) + int countActiveAnnouncement( + @Define("tableName") String tableName, @Bind("entityLink") String entityLink); + + @ConnectionAwareSqlQuery( + value = + "SELECT COUNT(te.id) AS count " + + "FROM thread_entity te " + + "WHERE te.type = 'Announcement' " + + " AND te.entityLink = :entityLink " + + " AND CAST(JSON_EXTRACT(te.json, '$.announcement.startTime') AS UNSIGNED) <= UNIX_TIMESTAMP()*1000 " + + " AND CAST(JSON_EXTRACT(te.json, '$.announcement.endTime') AS UNSIGNED) >= UNIX_TIMESTAMP()*1000", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT COUNT(te.id) AS count " + + "FROM thread_entity te " + + "WHERE te.type = 'Announcement' " + + " AND te.entityLink = :entityLink " + + " AND (te.json->'announcement'->>'startTime')::numeric <= EXTRACT(EPOCH FROM NOW()) * 1000 " + + " AND (te.json->'announcement'->>'endTime')::numeric >= EXTRACT(EPOCH FROM NOW()) * 1000", + connectionType = POSTGRES) + int countActiveAnnouncement(@Bind("entityLink") String entityLink); + + @ConnectionAwareSqlQuery( + value = + "SELECT combined.type, combined.taskStatus, COUNT(combined.id) AS count " + + "FROM ( " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " JOIN entity_relationship er ON te.entityId = er.toId " + + " WHERE " + + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 8 AND te.type <> 'Task') " + + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 8 AND te.type <> 'Task') " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " JOIN entity_relationship er ON te.id = er.toId " + + " WHERE " + + " er.fromEntity = 'user' AND er.fromId = :userId AND er.toEntity = 'THREAD' AND er.relation IN (1, 2) " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " JOIN entity_relationship er ON te.id = er.toId " + + " WHERE " + + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 11) " + + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 11) " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE te.createdBy = :username " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE MATCH(te.taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) " + + ") AS combined WHERE combined.type is not NULL " + + "GROUP BY combined.type, combined.taskStatus;", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT combined.type, combined.taskStatus, COUNT(combined.id) AS count " + + "FROM ( " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " JOIN entity_relationship er ON te.entityId = er.toId " + + " WHERE " + + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 8 AND te.type <> 'Task') " + + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 8 AND te.type <> 'Task') " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " JOIN entity_relationship er ON te.id = er.toId " + + " WHERE " + + " er.fromEntity = 'user' AND er.fromId = :userId AND er.toEntity = 'THREAD' AND er.relation IN (1, 2) " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " JOIN entity_relationship er ON te.id = er.toId " + + " WHERE " + + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 11) " + + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 11) " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE te.createdBy = :username " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) " + + ") AS combined WHERE combined.type is not NULL " + + "GROUP BY combined.type, combined.taskStatus;", + connectionType = POSTGRES) + @RegisterRowMapper(OwnerCountFieldMapper.class) + List> listCountByOwner( + @Define("tableName") String tableName, + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("username") String username, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres); + + @ConnectionAwareSqlQuery( + value = + "SELECT combined.type, combined.taskStatus, COUNT(combined.id) AS count " + + "FROM ( " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " JOIN entity_relationship er ON te.entityId = er.toId " + + " WHERE " + + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 8 AND te.type <> 'Task') " + + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 8 AND te.type <> 'Task') " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " JOIN entity_relationship er ON te.id = er.toId " + + " WHERE " + + " er.fromEntity = 'user' AND er.fromId = :userId AND er.toEntity = 'THREAD' AND er.relation IN (1, 2) " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " JOIN entity_relationship er ON te.id = er.toId " + + " WHERE " + + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 11) " + + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 11) " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE te.createdBy = :username " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE MATCH(te.taskAssigneesIds) AGAINST (:userTeamJsonMysql IN BOOLEAN MODE) " + + ") AS combined WHERE combined.type is not NULL " + + "GROUP BY combined.type, combined.taskStatus;", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT combined.type, combined.taskStatus, COUNT(combined.id) AS count " + + "FROM ( " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " JOIN entity_relationship er ON te.entityId = er.toId " + + " WHERE " + + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 8 AND te.type <> 'Task') " + + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 8 AND te.type <> 'Task') " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " JOIN entity_relationship er ON te.id = er.toId " + + " WHERE " + + " er.fromEntity = 'user' AND er.fromId = :userId AND er.toEntity = 'THREAD' AND er.relation IN (1, 2) " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " JOIN entity_relationship er ON te.id = er.toId " + + " WHERE " + + " (er.fromEntity = 'user' AND er.fromId = :userId AND er.relation = 11) " + + " OR (er.fromEntity = 'team' AND er.fromId IN () AND er.relation = 11) " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE te.createdBy = :username " + + " UNION " + + " SELECT te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE to_tsvector('simple', taskAssigneesIds) @@ to_tsquery('simple', :userTeamJsonPostgres) " + + ") AS combined WHERE combined.type is not NULL " + + "GROUP BY combined.type, combined.taskStatus;", + connectionType = POSTGRES) + @RegisterRowMapper(OwnerCountFieldMapper.class) + List> listCountByOwner( + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("username") String username, + @Bind("userTeamJsonMysql") String userTeamJsonMysql, + @Bind("userTeamJsonPostgres") String userTeamJsonPostgres); + + @SqlQuery( + "SELECT json FROM AND " + + "entityId in (" + + "SELECT toId FROM entity_relationship WHERE " + + "((fromEntity='user' AND fromId= :userId) OR " + + "(fromEntity='team' AND fromId IN ())) AND relation= :relation) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByFollows( + @Define("tableName") String tableName, + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("limit") int limit, + @Bind("relation") int relation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM thread_entity AND " + + "entityId in (" + + "SELECT toId FROM entity_relationship WHERE " + + "((fromEntity='user' AND fromId= :userId) OR " + + "(fromEntity='team' AND fromId IN ())) AND relation= :relation) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByFollows( + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("limit") int limit, + @Bind("relation") int relation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT count(id) FROM AND " + + "entityId in (" + + "SELECT toId FROM entity_relationship WHERE " + + "((fromEntity='user' AND fromId= :userId) OR " + + "(fromEntity='team' AND fromId IN ())) AND relation= :relation)") + int listCountThreadsByFollows( + @Define("tableName") String tableName, + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("relation") int relation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT count(id) FROM thread_entity AND " + + "entityId in (" + + "SELECT toId FROM entity_relationship WHERE " + + "((fromEntity='user' AND fromId= :userId) OR " + + "(fromEntity='team' AND fromId IN ())) AND relation= :relation)") + int listCountThreadsByFollows( + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("relation") int relation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM ( " + + " SELECT json, createdAt FROM te " + + " AND entityId IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.relation = 8 " + + " AND ( " + + " (er.fromEntity = 'user' AND er.fromId = :userId) " + + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " + + " ) " + + " ) " + + " UNION " + + " SELECT json, createdAt FROM te " + + " AND id IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.toEntity = 'THREAD' " + + " AND er.relation IN (1, 2) " + + " AND er.fromEntity = 'user' " + + " AND er.fromId = :userId " + + " ) " + + " UNION " + + " SELECT json, createdAt FROM te " + + " AND id IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.relation = 11 " + + " AND ( " + + " (er.fromEntity = 'user' AND er.fromId = :userId) " + + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " + + " ) " + + " ) " + + ") AS combined " + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByOwnerOrFollows( + @Define("tableName") String tableName, + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM ( " + + " SELECT json, createdAt FROM thread_entity te " + + " AND entityId IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.relation = 8 " + + " AND ( " + + " (er.fromEntity = 'user' AND er.fromId = :userId) " + + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " + + " ) " + + " ) " + + " UNION " + + " SELECT json, createdAt FROM thread_entity te " + + " AND id IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.toEntity = 'THREAD' " + + " AND er.relation IN (1, 2) " + + " AND er.fromEntity = 'user' " + + " AND er.fromId = :userId " + + " ) " + + " UNION " + + " SELECT json, createdAt FROM thread_entity te " + + " AND id IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.relation = 11 " + + " AND ( " + + " (er.fromEntity = 'user' AND er.fromId = :userId) " + + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " + + " ) " + + " ) " + + ") AS combined " + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByOwnerOrFollows( + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT COUNT(id) FROM ( " + + " SELECT te.id FROM te " + + " AND entityId IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.relation = 8 " + + " AND ( " + + " (er.fromEntity = 'user' AND er.fromId = :userId) " + + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " + + " ) " + + " ) " + + " UNION " + + " SELECT te.id FROM te " + + " AND id IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.toEntity = 'THREAD' " + + " AND er.relation IN (1, 2) " + + " AND er.fromEntity = 'user' " + + " AND er.fromId = :userId " + + " ) " + + " UNION " + + " SELECT te.id FROM te " + + " AND id IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.relation = 11 " + + " AND ( " + + " (er.fromEntity = 'user' AND er.fromId = :userId) " + + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " + + " ) " + + " ) " + + ") AS combined") + int listCountThreadsByOwnerOrFollows( + @Define("tableName") String tableName, + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT COUNT(id) FROM ( " + + " SELECT te.id FROM thread_entity te " + + " AND entityId IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.relation = 8 " + + " AND ( " + + " (er.fromEntity = 'user' AND er.fromId = :userId) " + + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " + + " ) " + + " ) " + + " UNION " + + " SELECT te.id FROM thread_entity te " + + " AND id IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.toEntity = 'THREAD' " + + " AND er.relation IN (1, 2) " + + " AND er.fromEntity = 'user' " + + " AND er.fromId = :userId " + + " ) " + + " UNION " + + " SELECT te.id FROM thread_entity te " + + " AND id IN ( " + + " SELECT toId FROM entity_relationship er " + + " WHERE er.relation = 11 " + + " AND ( " + + " (er.fromEntity = 'user' AND er.fromId = :userId) " + + " OR (er.fromEntity = 'team' AND er.fromId IN ()) " + + " ) " + + " ) " + + ") AS combined") + int listCountThreadsByOwnerOrFollows( + @BindUUID("userId") UUID userId, + @BindList("teamIds") List teamIds, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM AND " + + "hash_id in (" + + "SELECT toFQNHash FROM field_relationship WHERE " + + "((fromType='user' AND fromFQNHash= :userName) OR " + + "(fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :relation) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByMentions( + @Define("tableName") String tableName, + @Bind("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("limit") int limit, + @Bind("relation") int relation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM thread_entity AND " + + "hash_id in (" + + "SELECT toFQNHash FROM field_relationship WHERE " + + "((fromType='user' AND fromFQNHash= :userName) OR " + + "(fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :relation) " + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByMentions( + @Bind("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("limit") int limit, + @Bind("relation") int relation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT count(id) FROM AND " + + "hash_id in (" + + "SELECT toFQNHash FROM field_relationship WHERE " + + "((fromType='user' AND fromFQNHash= :userName) OR " + + "(fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :relation) ") + int listCountThreadsByMentions( + @Define("tableName") String tableName, + @Bind("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("relation") int relation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT count(id) FROM thread_entity AND " + + "hash_id in (" + + "SELECT toFQNHash FROM field_relationship WHERE " + + "((fromType='user' AND fromFQNHash= :userName) OR " + + "(fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :relation) ") + int listCountThreadsByMentions( + @Bind("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("relation") int relation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM " + + "AND MD5(id) in (SELECT fromFQNHash FROM field_relationship WHERE " + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "((:toType1 IS NULL OR toType LIKE :concatToType1 OR toType=:toType1) OR " + + "(:toType2 IS NULL OR toType LIKE :concatToType2 OR toType=:toType2)) AND relation= :relation)" + + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " + + " ((fromType='user' AND fromFQNHash= :userName) OR" + + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByGlossaryAndTerms( + @Define("tableName") String tableName, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType1", + original = "toType1", + parts = {":toType1", ".%"}) + String toType1, + @BindConcat( + value = "concatToType2", + original = "toType2", + parts = {":toType2", ".%"}) + String toType2, + @Bind("limit") int limit, + @Bind("relation") int relation, + @BindFQN("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("filterRelation") int filterRelation, + @Define("condition") String condition, + @BindMap Map params); + + @SqlQuery( + "SELECT json FROM thread_entity " + + "AND MD5(id) in (SELECT fromFQNHash FROM field_relationship WHERE " + + "(:fqnPrefixHash IS NULL OR toFQNHash LIKE :concatFqnPrefixHash OR toFQNHash=:fqnPrefixHash) AND fromType='THREAD' AND " + + "((:toType1 IS NULL OR toType LIKE :concatToType1 OR toType=:toType1) OR " + + "(:toType2 IS NULL OR toType LIKE :concatToType2 OR toType=:toType2)) AND relation= :relation)" + + "AND (:userName IS NULL OR MD5(id) in (SELECT toFQNHash FROM field_relationship WHERE " + + " ((fromType='user' AND fromFQNHash= :userName) OR" + + " (fromType='team' AND fromFQNHash IN ())) AND toType='THREAD' AND relation= :filterRelation) )" + + "ORDER BY createdAt DESC " + + "LIMIT :limit") + List listThreadsByGlossaryAndTerms( + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType1", + original = "toType1", + parts = {":toType1", ".%"}) + String toType1, + @BindConcat( + value = "concatToType2", + original = "toType2", + parts = {":toType2", ".%"}) + String toType2, + @Bind("limit") int limit, + @Bind("relation") int relation, + @BindFQN("userName") String userName, + @BindList("teamNames") List teamNames, + @Bind("filterRelation") int filterRelation, + @Define("condition") String condition, + @BindMap Map params); + + default List> listCountThreadsByGlossaryAndTerms( + String tableName, EntityLink entityLink, EntityReference reference) { + EntityLink glossaryTermLink = + new EntityLink(GLOSSARY_TERM, entityLink.getFullyQualifiedFieldValue()); + return listCountThreadsByGlossaryAndTerms( + tableName, + reference.getId(), + reference.getFullyQualifiedName(), + entityLink.getFullyQualifiedFieldType(), + glossaryTermLink.getFullyQualifiedFieldType()); + } + + default List> listCountThreadsByGlossaryAndTerms( + EntityLink entityLink, EntityReference reference) { + EntityLink glossaryTermLink = + new EntityLink(GLOSSARY_TERM, entityLink.getFullyQualifiedFieldValue()); + return listCountThreadsByGlossaryAndTerms( + reference.getId(), + reference.getFullyQualifiedName(), + entityLink.getFullyQualifiedFieldType(), + glossaryTermLink.getFullyQualifiedFieldType()); + } + + default List listThreadsByTaskAssignee(String taskAssigneesId) { + return listThreadsByTaskAssigneesId("%" + taskAssigneesId + "%"); + } + + @SqlQuery("SELECT json FROM WHERE taskAssigneesIds LIKE :taskAssigneesPattern") + List listThreadsByTaskAssigneesId( + @Define("tableName") String tableName, + @Bind("taskAssigneesPattern") String taskAssigneesPattern); + + @SqlQuery("SELECT json FROM thread_entity WHERE taskAssigneesIds LIKE :taskAssigneesPattern") + List listThreadsByTaskAssigneesId( + @Bind("taskAssigneesPattern") String taskAssigneesPattern); + + @SqlQuery( + "SELECT entityLink, type, taskStatus, COUNT(id) as count " + + "FROM ( " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE te.entityId = :entityId " + + " UNION " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE te.hash_id IN ( " + + " SELECT fr.fromFQNHash " + + " FROM field_relationship fr " + + " WHERE (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " + + " AND fr.fromType = 'THREAD' " + + " AND (:toType1 IS NULL OR fr.toType LIKE :concatToType1 OR fr.toType = :toType1) " + + " AND fr.relation = 3 " + + " ) " + + " UNION " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM te " + + " WHERE te.type = 'Task' " + + " AND te.hash_id IN ( " + + " SELECT fr.fromFQNHash " + + " FROM field_relationship fr " + + " JOIN te2 ON te2.hash_id = fr.fromFQNHash WHERE fr.fromFQNHash = te.hash_id AND te2.type = 'Task' " + + " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " + + " AND fr.fromType = 'THREAD' " + + " AND (:toType2 IS NULL OR fr.toType LIKE :concatToType2 OR fr.toType = :toType2) " + + " AND fr.relation = 3 " + + " ) " + + ") AS combined_results WHERE combined_results.type is not NULL " + + "GROUP BY entityLink, type, taskStatus ") + @RegisterRowMapper(ThreadCountFieldMapper.class) + List> listCountThreadsByGlossaryAndTerms( + @Define("tableName") String tableName, + @BindUUID("entityId") UUID entityId, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType1", + original = "toType1", + parts = {":toType1", ".%"}) + String toType1, + @BindConcat( + value = "concatToType2", + original = "toType2", + parts = {":toType2", ".%"}) + String toType2); + + @SqlQuery( + "SELECT entityLink, type, taskStatus, COUNT(id) as count " + + "FROM ( " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE te.entityId = :entityId " + + " UNION " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE te.hash_id IN ( " + + " SELECT fr.fromFQNHash " + + " FROM field_relationship fr " + + " WHERE (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " + + " AND fr.fromType = 'THREAD' " + + " AND (:toType1 IS NULL OR fr.toType LIKE :concatToType1 OR fr.toType = :toType1) " + + " AND fr.relation = 3 " + + " ) " + + " UNION " + + " SELECT te.entityLink, te.type, te.taskStatus, te.id " + + " FROM thread_entity te " + + " WHERE te.type = 'Task' " + + " AND te.hash_id IN ( " + + " SELECT fr.fromFQNHash " + + " FROM field_relationship fr " + + " JOIN thread_entity te2 ON te2.hash_id = fr.fromFQNHash WHERE fr.fromFQNHash = te.hash_id AND te2.type = 'Task' " + + " AND (:fqnPrefixHash IS NULL OR fr.toFQNHash LIKE :concatFqnPrefixHash OR fr.toFQNHash = :fqnPrefixHash) " + + " AND fr.fromType = 'THREAD' " + + " AND (:toType2 IS NULL OR fr.toType LIKE :concatToType2 OR fr.toType = :toType2) " + + " AND fr.relation = 3 " + + " ) " + + ") AS combined_results WHERE combined_results.type is not NULL " + + "GROUP BY entityLink, type, taskStatus ") + @RegisterRowMapper(ThreadCountFieldMapper.class) + List> listCountThreadsByGlossaryAndTerms( + @BindUUID("entityId") UUID entityId, + @BindConcat( + value = "concatFqnPrefixHash", + original = "fqnPrefixHash", + parts = {":fqnPrefixHash", ".%"}, + hash = true) + String fqnPrefixHash, + @BindConcat( + value = "concatToType1", + original = "toType1", + parts = {":toType1", ".%"}) + String toType1, + @BindConcat( + value = "concatToType2", + original = "toType2", + parts = {":toType2", ".%"}) + String toType2); + + @SqlQuery("select id from where entityId = :entityId") + List findByEntityId( + @Define("tableName") String tableName, @Bind("entityId") String entityId); + + @SqlQuery("select id from thread_entity where entityId = :entityId") + List findByEntityId(@Bind("entityId") String entityId); + + // DISTINCT is defence-in-depth: thread_entity.id is a primary key, and entityId is a + // single-valued column per row, so a single matching scan can't physically return the + // same id twice. The DISTINCT survives a future schema where a thread row picks up + // multiple entity references (or a join is added) — keeping the consumer code in + // deleteByAbout from re-issuing redundant relationship / extension / feed deletes for + // the same id under chunking. + @SqlQuery("select DISTINCT id from where entityId IN ()") + List findByEntityIds( + @Define("tableName") String tableName, @BindList("entityIds") List entityIds); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE SET json = JSON_SET(json, '$.about', :newEntityLink)\n" + + "WHERE entityId = :entityId", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE SET json = jsonb_set(json, '{about}', to_jsonb(:newEntityLink::text), false)\n" + + "WHERE entityId = :entityId", + connectionType = POSTGRES) + void updateByEntityId( + @Define("tableName") String tableName, + @Bind("newEntityLink") String newEntityLink, + @Bind("entityId") String entityId); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE thread_entity SET json = JSON_SET(json, '$.about', :newEntityLink)\n" + + "WHERE entityId = :entityId", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE thread_entity SET json = jsonb_set(json, '{about}', to_jsonb(:newEntityLink::text), false)\n" + + "WHERE entityId = :entityId", + connectionType = POSTGRES) + void updateByEntityId( + @Bind("newEntityLink") String newEntityLink, @Bind("entityId") String entityId); + + class OwnerCountFieldMapper implements RowMapper> { + @Override + public List map(ResultSet rs, StatementContext ctx) throws SQLException { + return Arrays.asList( + rs.getString("type"), rs.getString("taskStatus"), rs.getString("count")); + } + } + + class ThreadCountFieldMapper implements RowMapper> { + @Override + public List map(ResultSet rs, StatementContext ctx) throws SQLException { + return Arrays.asList( + rs.getString("entityLink"), + rs.getString("type"), + rs.getString("taskStatus"), + rs.getString("count")); + } + } + } + + interface TaskDAO extends EntityDAO { + class TaskCountSummary { + private final int total; + private final int open; + private final int completed; + private final int inProgress; + private final int approved; + private final int granted; + + public TaskCountSummary( + int total, int open, int completed, int inProgress, int approved, int granted) { + this.total = total; + this.open = open; + this.completed = completed; + this.inProgress = inProgress; + this.approved = approved; + this.granted = granted; + } + + public int getTotal() { + return total; + } + + public int getOpen() { + return open; + } + + public int getCompleted() { + return completed; + } + + public int getInProgress() { + return inProgress; + } + + public int getApproved() { + return approved; + } + + public int getGranted() { + return granted; + } + } + + class TaskCountSummaryMapper implements RowMapper { + @Override + public TaskCountSummary map(ResultSet rs, StatementContext ctx) throws SQLException { + return new TaskCountSummary( + rs.getInt("total"), + rs.getInt("openCount"), + rs.getInt("completedCount"), + rs.getInt("inProgressCount"), + rs.getInt("approvedCount"), + rs.getInt("grantedCount")); + } + } + + @Override + default String getTableName() { + return "task_entity"; + } + + @Override + default Class getEntityClass() { + return Task.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @ConnectionAwareSqlUpdate( + value = "INSERT INTO task_entity (id, json, fqnHash) VALUES (:id, :json, :fqnHash)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO task_entity (id, json, fqnHash) VALUES (:id, :json :: jsonb, :fqnHash)", + connectionType = POSTGRES) + void insertTask( + @Bind("id") String id, @Bind("json") String json, @BindFQN("fqnHash") String fqn); + + @Override + default void insert(org.openmetadata.schema.EntityInterface entity, String fqn) { + Task task = (Task) entity; + insertTask(task.getId().toString(), JsonUtils.pojoToJson(task), task.getFullyQualifiedName()); + } + + @ConnectionAwareSqlUpdate( + value = "UPDATE task_entity SET json = :json WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE task_entity SET json = (:json :: jsonb) WHERE id = :id", + connectionType = POSTGRES) + void updateTask(@Bind("id") String id, @Bind("json") String json); + + @Override + default void update(UUID id, String fqn, String json) { + updateTask(id.toString(), json); + } + + @SqlUpdate("UPDATE new_task_sequence SET id = LAST_INSERT_ID(id + 1)") + int incrementSequenceMysql(); + + @SqlQuery("SELECT LAST_INSERT_ID()") + long getLastInsertIdMysql(); + + @SqlQuery("UPDATE new_task_sequence SET id = id + 1 RETURNING id") + long getNextTaskIdPostgres(); + + @SqlUpdate("DELETE FROM entity_relationship WHERE fromEntity = 'task' OR toEntity = 'task'") + void deleteTaskRelationships(); + + @SqlUpdate("DELETE FROM task_entity") + void deleteAll(); + + @SqlUpdate("UPDATE new_task_sequence SET id = 0") + void resetSequence(); + + @SqlUpdate( + "DELETE FROM entity_relationship WHERE fromEntity = 'domain' AND toEntity = 'task' " + + "AND relation = 10 AND toId IN ()") + void bulkRemoveDomainRelationships(@BindList("taskIds") List taskIds); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM task_entity " + + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.payload.testCaseResolutionStatusId')) = :stateId " + + "AND (JSON_EXTRACT(json, '$.deleted') = false OR JSON_EXTRACT(json, '$.deleted') IS NULL)", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM task_entity " + + "WHERE json->'payload'->>'testCaseResolutionStatusId' = :stateId " + + "AND ((json->>'deleted')::boolean = false OR json->>'deleted' IS NULL)", + connectionType = POSTGRES) + String fetchTaskByTestCaseResolutionStatusId(@Bind("stateId") String stateId); + + @SqlQuery( + "SELECT json FROM task_entity " + + "WHERE aboutFqnHash = :aboutFqnHash AND type = :type " + + "AND status IN () " + + "AND (deleted = false OR deleted IS NULL) " + + "ORDER BY createdAt DESC LIMIT 1") + String findByAboutAndTypeAndStatuses( + @BindFQN("aboutFqnHash") String aboutFqn, + @Bind("type") String type, + @BindList("statuses") List statuses); + + @SqlQuery( + "SELECT json FROM task_entity " + + "WHERE aboutFqnHash = :aboutFqnHash AND type = :type AND status = :status " + + "AND (deleted = false OR deleted IS NULL) " + + "LIMIT 1") + String findByAboutAndTypeAndStatus( + @BindFQN("aboutFqnHash") String aboutFqn, + @Bind("type") String type, + @Bind("status") String status); + + @SqlQuery( + "SELECT json FROM task_entity " + + "WHERE aboutFqnHash = :aboutFqnHash AND category = :category AND status = :status " + + "AND (deleted = false OR deleted IS NULL) " + + "LIMIT 1") + String findByAboutAndCategoryAndStatus( + @BindFQN("aboutFqnHash") String aboutFqn, + @Bind("category") String category, + @Bind("status") String status); + + @SqlUpdate( + "DELETE FROM task_entity " + "WHERE createdById = :createdById AND category = :category") + void deleteByCreatorAndCategory( + @Bind("createdById") String createdById, @Bind("category") String category); + + @ConnectionAwareSqlQuery( + value = + "SELECT id, json_unquote(json_extract(json, '$.fullyQualifiedName')) AS fqn " + + "FROM task_entity WHERE createdById = :createdById AND category = :category", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT id, json->>'fullyQualifiedName' AS fqn " + + "FROM task_entity WHERE createdById = :createdById AND category = :category", + connectionType = POSTGRES) + @RegisterRowMapper(EntityDAO.EntityIdFqnPairMapper.class) + List listIdAndFqnByCreatorAndCategory( + @Bind("createdById") String createdById, @Bind("category") String category); + + @RegisterRowMapper(TaskCountSummaryMapper.class) + @SqlQuery( + // 'Approved' double-counts in `completedCount` AND `approvedCount` because the + // same status means different things across task types: terminal for + // Glossary/DescriptionUpdate (legacy dashboards expect it under "completed") and + // non-terminal for Data Access Requests (the dedicated DAR list uses + // `approvedCount` / `grantedCount` and the `active` status group instead). + // See ListFilter.getTaskStatusCondition for the matching status-group semantics. + "SELECT " + + "COUNT(id) AS total, " + + "COALESCE(SUM(CASE WHEN status IN ('Open', 'InProgress', 'Pending') THEN 1 ELSE 0 END), 0) AS openCount, " + + "COALESCE(SUM(CASE WHEN status IN ('Approved', 'Rejected', 'Completed', 'Cancelled', 'Failed', 'Revoked') THEN 1 ELSE 0 END), 0) AS completedCount, " + + "COALESCE(SUM(CASE WHEN status = 'InProgress' THEN 1 ELSE 0 END), 0) AS inProgressCount, " + + "COALESCE(SUM(CASE WHEN status = 'Approved' THEN 1 ELSE 0 END), 0) AS approvedCount, " + + "COALESCE(SUM(CASE WHEN status = 'Granted' THEN 1 ELSE 0 END), 0) AS grantedCount " + + "FROM task_entity ") + TaskCountSummary getTaskCountSummary( + @Define("condition") String condition, @BindMap Map params); + + @SqlQuery( + "SELECT json FROM task_entity " + + "ORDER BY createdAt , id " + + "LIMIT :limit OFFSET :offset") + List listTasksByCreatedAt( + @Define("cond") String cond, + @BindMap Map params, + @Define("sortOrder") String sortOrder, + @Bind("limit") int limit, + @Bind("offset") int offset); + + @SqlQuery("SELECT count(*) FROM task_entity ") + int listTasksByCreatedAtCount(@Define("cond") String cond, @BindMap Map params); + } + + interface AnnouncementDAO extends EntityDAO { + @Override + default String getTableName() { + return "announcement_entity"; + } + + @Override + default Class getEntityClass() { + return Announcement.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @ConnectionAwareSqlUpdate( + value = "INSERT INTO announcement_entity (id, json, fqnHash) VALUES (:id, :json, :fqnHash)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO announcement_entity (id, json, fqnHash) VALUES (:id, :json :: jsonb, :fqnHash)", + connectionType = POSTGRES) + void insertAnnouncement( + @Bind("id") String id, @Bind("json") String json, @BindFQN("fqnHash") String fqn); + + @Override + default void insert(org.openmetadata.schema.EntityInterface entity, String fqn) { + Announcement announcement = (Announcement) entity; + insertAnnouncement( + announcement.getId().toString(), + JsonUtils.pojoToJson(announcement), + announcement.getFullyQualifiedName()); + } + + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM announcement_entity " + + "WHERE " + + "AND (:entityLink IS NULL OR entityLink = :entityLink) " + + "AND (:status IS NULL OR status = :status) " + + "AND ((:active IS NULL) " + + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " + + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs)))", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT count(*) FROM announcement_entity " + + "WHERE " + + "AND (:entityLink IS NULL OR entityLink = :entityLink) " + + "AND (:status IS NULL OR status = :status) " + + "AND ((:active IS NULL) " + + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " + + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs)))", + connectionType = POSTGRES) + int listAnnouncementCount( + @Define("condition") String condition, + @Bind("entityLink") String entityLink, + @Bind("status") String status, + @Bind("active") Boolean active, + @Bind("currentTs") long currentTs); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM announcement_entity " + + "WHERE " + + "AND (:entityLink IS NULL OR entityLink = :entityLink) " + + "AND (:status IS NULL OR status = :status) " + + "AND ((:active IS NULL) " + + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " + + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " + + "ORDER BY name, id LIMIT :limit OFFSET :offset", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM announcement_entity " + + "WHERE " + + "AND (:entityLink IS NULL OR entityLink = :entityLink) " + + "AND (:status IS NULL OR status = :status) " + + "AND ((:active IS NULL) " + + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " + + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " + + "ORDER BY name, id LIMIT :limit OFFSET :offset", + connectionType = POSTGRES) + List listAnnouncementsWithOffset( + @Define("condition") String condition, + @Bind("entityLink") String entityLink, + @Bind("status") String status, + @Bind("active") Boolean active, + @Bind("currentTs") long currentTs, + @Bind("limit") int limit, + @Bind("offset") int offset); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT announcement_entity.name, announcement_entity.id, announcement_entity.json " + + "FROM announcement_entity " + + "WHERE " + + "AND (:entityLink IS NULL OR entityLink = :entityLink) " + + "AND (:status IS NULL OR status = :status) " + + "AND ((:active IS NULL) " + + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " + + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " + + "AND (announcement_entity.name < :beforeName " + + "OR (announcement_entity.name = :beforeName AND announcement_entity.id < :beforeId)) " + + "ORDER BY announcement_entity.name DESC, announcement_entity.id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name, id", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT announcement_entity.name, announcement_entity.id, announcement_entity.json " + + "FROM announcement_entity " + + "WHERE " + + "AND (:entityLink IS NULL OR entityLink = :entityLink) " + + "AND (:status IS NULL OR status = :status) " + + "AND ((:active IS NULL) " + + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " + + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " + + "AND (announcement_entity.name < :beforeName " + + "OR (announcement_entity.name = :beforeName AND announcement_entity.id < :beforeId)) " + + "ORDER BY announcement_entity.name DESC, announcement_entity.id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name, id", + connectionType = POSTGRES) + List listAnnouncementsBefore( + @Define("condition") String condition, + @Bind("entityLink") String entityLink, + @Bind("status") String status, + @Bind("active") Boolean active, + @Bind("currentTs") long currentTs, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId); + + @ConnectionAwareSqlQuery( + value = + "SELECT announcement_entity.json FROM announcement_entity " + + "WHERE " + + "AND (:entityLink IS NULL OR entityLink = :entityLink) " + + "AND (:status IS NULL OR status = :status) " + + "AND ((:active IS NULL) " + + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " + + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " + + "AND (announcement_entity.name > :afterName " + + "OR (announcement_entity.name = :afterName AND announcement_entity.id > :afterId)) " + + "ORDER BY announcement_entity.name, announcement_entity.id " + + "LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT announcement_entity.json FROM announcement_entity " + + "WHERE " + + "AND (:entityLink IS NULL OR entityLink = :entityLink) " + + "AND (:status IS NULL OR status = :status) " + + "AND ((:active IS NULL) " + + "OR (:active = TRUE AND startTime <= :currentTs AND endTime >= :currentTs) " + + "OR (:active = FALSE AND (startTime > :currentTs OR endTime < :currentTs))) " + + "AND (announcement_entity.name > :afterName " + + "OR (announcement_entity.name = :afterName AND announcement_entity.id > :afterId)) " + + "ORDER BY announcement_entity.name, announcement_entity.id " + + "LIMIT :limit", + connectionType = POSTGRES) + List listAnnouncementsAfter( + @Define("condition") String condition, + @Bind("entityLink") String entityLink, + @Bind("status") String status, + @Bind("active") Boolean active, + @Bind("currentTs") long currentTs, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId); + + private String getAnnouncementBaseCondition(ListFilter filter) { + String includeCondition = filter.getIncludeCondition(getTableName()); + return includeCondition.isEmpty() ? "TRUE" : includeCondition; + } + + private Boolean getActiveFlag(ListFilter filter) { + String active = filter.getQueryParam("active"); + return active == null ? null : Boolean.parseBoolean(active); + } + + private String getAnnouncementStatus(ListFilter filter) { + return filter.getQueryParam("status"); + } + + private String getAnnouncementEntityLink(ListFilter filter) { + return filter.getQueryParam("entityLink"); + } + + @Override + default int listCount(ListFilter filter) { + if (filter.getQueryParam("active") == null) { + return EntityDAO.super.listCount(filter); + } + + return listAnnouncementCount( + getAnnouncementBaseCondition(filter), + getAnnouncementEntityLink(filter), + getAnnouncementStatus(filter), + getActiveFlag(filter), + System.currentTimeMillis()); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + if (filter.getQueryParam("active") == null) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + + return listAnnouncementsBefore( + getAnnouncementBaseCondition(filter), + getAnnouncementEntityLink(filter), + getAnnouncementStatus(filter), + getActiveFlag(filter), + System.currentTimeMillis(), + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + if (filter.getQueryParam("active") == null) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + + return listAnnouncementsAfter( + getAnnouncementBaseCondition(filter), + getAnnouncementEntityLink(filter), + getAnnouncementStatus(filter), + getActiveFlag(filter), + System.currentTimeMillis(), + limit, + afterName, + afterId); + } + + @Override + default List listAfter(ListFilter filter, int limit, int offset) { + if (filter.getQueryParam("active") == null) { + return EntityDAO.super.listAfter(filter, limit, offset); + } + + return listAnnouncementsWithOffset( + getAnnouncementBaseCondition(filter), + getAnnouncementEntityLink(filter), + getAnnouncementStatus(filter), + getActiveFlag(filter), + System.currentTimeMillis(), + limit, + offset); + } + } + + interface TaskFormSchemaDAO extends EntityDAO { + @Override + default String getTableName() { + return "task_form_schema_entity"; + } + + @Override + default Class getEntityClass() { + return TaskFormSchema.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO task_form_schema_entity (id, json, fqnHash) VALUES (:id, :json, :fqnHash)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO task_form_schema_entity (id, json, fqnHash) VALUES (:id, :json :: jsonb, :fqnHash)", + connectionType = POSTGRES) + void insertTaskFormSchema( + @Bind("id") String id, @Bind("json") String json, @BindFQN("fqnHash") String fqn); + + @Override + default void insert(org.openmetadata.schema.EntityInterface entity, String fqn) { + TaskFormSchema schema = (TaskFormSchema) entity; + insertTaskFormSchema( + schema.getId().toString(), JsonUtils.pojoToJson(schema), schema.getFullyQualifiedName()); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java index fc697bdd8364..ba87dab83575 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryRepository.java @@ -78,7 +78,7 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.glossary.GlossaryResource; import org.openmetadata.service.resources.settings.SettingsCache; @@ -683,9 +683,9 @@ public void updateName(Glossary updated) { // Capture the descendants so the post-write pass can re-evict any entry a racing reader // re-populated with the pre-rename row between this call and glossaryTermDAO.updateFqn. // The pass below runs after updateFqn but inside this transaction — see - // EntityRepository.invalidateCacheForRenameCascade for the residual pre-commit window. + // EntityCacheInvalidator.invalidateCacheForRenameCascade for the residual pre-commit window. List renamedTerms = - invalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, oldFqn); + EntityCacheInvalidator.invalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, oldFqn); daoCollection.glossaryTermDAO().updateFqn(oldFqn, newFqn); daoCollection.tagUsageDAO().updateTagPrefix(TagSource.GLOSSARY.ordinal(), oldFqn, newFqn); recordChange("name", FullyQualifiedName.unquoteName(oldFqn), updated.getName()); @@ -706,12 +706,13 @@ public void updateName(Glossary updated) { PolicyConditionUpdater.renamePrefixInCondition( condition, oldFqn, newFqn, PolicyConditionUpdater.TAG_FUNCTIONS)); - finishInvalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, renamedTerms); + EntityCacheInvalidator.finishInvalidateCacheForRenameCascade( + Entity.GLOSSARY_TERM, renamedTerms); } public void invalidateGlossary(UUID classificationId) { // Glossary name changed. Invalidate the glossary and its children terms - CACHE_WITH_ID.invalidate(new ImmutablePair<>(GLOSSARY, classificationId)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(GLOSSARY, classificationId)); List tags = findToRecords(classificationId, GLOSSARY, Relationship.CONTAINS, GLOSSARY_TERM); for (EntityRelationshipRecord tagRecord : tags) { @@ -724,7 +725,7 @@ private void invalidateTerms(UUID termId) { // children from the cache List tagRecords = findToRecords(termId, GLOSSARY_TERM, Relationship.CONTAINS, GLOSSARY_TERM); - CACHE_WITH_ID.invalidate(new ImmutablePair<>(GLOSSARY_TERM, termId)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(GLOSSARY_TERM, termId)); for (EntityRelationshipRecord tagRecord : tagRecords) { invalidateTerms(tagRecord.getId()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 33446ec3154b..880e6827618f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -108,7 +108,7 @@ import org.openmetadata.service.exception.BadRequestException; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.jdbi3.FeedRepository.TaskWorkflow; import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext; import org.openmetadata.service.rdf.RdfUpdater; @@ -2229,12 +2229,13 @@ public void updateNameAndParent(GlossaryTerm updated) { // Capture the descendants so the post-write pass can re-evict any entry a racing reader // re-populated with the pre-rename row between this call and glossaryTermDAO.updateFqn. // The pass below runs after updateFqn but inside this transaction — see - // EntityRepository.invalidateCacheForRenameCascade for the residual pre-commit window. + // EntityCacheInvalidator.invalidateCacheForRenameCascade for the residual pre-commit window. List renamedTerms = - invalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, oldFqn); + EntityCacheInvalidator.invalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, oldFqn); // Drop cached entity JSON / bundle for every entity tagged with this term (or any // descendant). Done BEFORE the DB rename so the search lookup still matches by old FQN. - invalidateCacheForTaggedEntitiesAndDescendants(Entity.GLOSSARY_TERM, oldFqn); + EntityCacheInvalidator.invalidateCacheForTaggedEntitiesAndDescendants( + Entity.GLOSSARY_TERM, oldFqn); daoCollection.glossaryTermDAO().updateFqn(oldFqn, newFqn); daoCollection.tagUsageDAO().rename(TagSource.GLOSSARY.ordinal(), oldFqn, newFqn); @@ -2276,7 +2277,8 @@ public void updateNameAndParent(GlossaryTerm updated) { updateAssetIndexes(oldFqn, newFqn); } - finishInvalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, renamedTerms); + EntityCacheInvalidator.finishInvalidateCacheForRenameCascade( + Entity.GLOSSARY_TERM, renamedTerms); } /** @@ -2305,12 +2307,13 @@ private void updateParent(GlossaryTerm original, GlossaryTerm updated) { // Capture the descendants so the post-write pass can re-evict any entry a racing reader // re-populated with the pre-rename row between this call and glossaryTermDAO.updateFqn. // The pass below runs after updateFqn but inside this transaction — see - // EntityRepository.invalidateCacheForRenameCascade for the residual pre-commit window. + // EntityCacheInvalidator.invalidateCacheForRenameCascade for the residual pre-commit window. List renamedTerms = - invalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, oldFqn); + EntityCacheInvalidator.invalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, oldFqn); // Drop cached entity JSON / bundle for every entity tagged with this term (or any // descendant). Done BEFORE the DB rename so the search lookup still matches by old FQN. - invalidateCacheForTaggedEntitiesAndDescendants(Entity.GLOSSARY_TERM, oldFqn); + EntityCacheInvalidator.invalidateCacheForTaggedEntitiesAndDescendants( + Entity.GLOSSARY_TERM, oldFqn); daoCollection.glossaryTermDAO().updateFqn(oldFqn, newFqn); daoCollection.tagUsageDAO().rename(TagSource.GLOSSARY.ordinal(), oldFqn, newFqn); @@ -2342,7 +2345,8 @@ private void updateParent(GlossaryTerm original, GlossaryTerm updated) { } updateAssetIndexes(oldFqn, newFqn); - finishInvalidateCacheForRenameCascade(Entity.GLOSSARY_TERM, renamedTerms); + EntityCacheInvalidator.finishInvalidateCacheForRenameCascade( + Entity.GLOSSARY_TERM, renamedTerms); } private void validateParent() { @@ -2422,7 +2426,7 @@ private void invalidateTerm(UUID termId, Set visited) { visited.add(termId); List tagRecords = findToRecords(termId, GLOSSARY_TERM, Relationship.CONTAINS, GLOSSARY_TERM); - CACHE_WITH_ID.invalidate(new ImmutablePair<>(GLOSSARY_TERM, termId)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(GLOSSARY_TERM, termId)); for (EntityRelationshipRecord tagRecord : tagRecords) { invalidateTerm(tagRecord.getId(), visited); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java new file mode 100644 index 000000000000..c61bb2891d1b --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java @@ -0,0 +1,214 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.util.List; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.openmetadata.schema.entity.data.DataContract; +import org.openmetadata.schema.entity.domains.DataProduct; +import org.openmetadata.schema.entity.domains.Domain; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.jdbi.BindConcat; + +public interface GovernanceDAOs { + @CreateSqlObject + DomainDAO domainDAO(); + + @CreateSqlObject + DataProductDAO dataProductDAO(); + + @CreateSqlObject + DataContractDAO dataContractDAO(); + + interface DomainDAO extends EntityDAO { + @Override + default String getTableName() { + return "domain_entity"; + } + + @Override + default Class getEntityClass() { + return Domain.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + + @Override + default int listCount(ListFilter filter) { + String condition = filter.getCondition(); + String directChildrenOf = filter.getQueryParam("directChildrenOf"); + String hierarchyFilter = filter.getQueryParam("hierarchyFilter"); + + if (!nullOrEmpty(directChildrenOf)) { + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; + } else if (Boolean.TRUE.toString().equals(hierarchyFilter)) { + // For hierarchy API, when directChildrenOf is null, show only root domains + condition += + " AND NOT EXISTS (SELECT 1 FROM entity_relationship er WHERE er.toId = domain_entity.id AND er.fromEntity = 'domain' AND er.toEntity = 'domain' AND er.relation = " + + Relationship.CONTAINS.ordinal() + + ")"; + } + + return listCount(getTableName(), getNameHashColumn(), filter.getQueryParams(), condition); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String condition = filter.getCondition(); + String directChildrenOf = filter.getQueryParam("directChildrenOf"); + String hierarchyFilter = filter.getQueryParam("hierarchyFilter"); + + if (!nullOrEmpty(directChildrenOf)) { + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; + } else if (Boolean.TRUE.toString().equals(hierarchyFilter)) { + // For hierarchy API, when directChildrenOf is null, show only root domains + condition += + " AND NOT EXISTS (SELECT 1 FROM entity_relationship er WHERE er.toId = domain_entity.id AND er.fromEntity = 'domain' AND er.toEntity = 'domain' AND er.relation = " + + Relationship.CONTAINS.ordinal() + + ")"; + } + + return listBefore( + getTableName(), filter.getQueryParams(), condition, limit, beforeName, beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String condition = filter.getCondition(); + String directChildrenOf = filter.getQueryParam("directChildrenOf"); + String hierarchyFilter = filter.getQueryParam("hierarchyFilter"); + String offsetParam = filter.getQueryParam("offset"); + + if (!nullOrEmpty(directChildrenOf)) { + String parentFqnHash = FullyQualifiedName.buildHash(directChildrenOf); + filter.queryParams.put("fqnHashSingleLevel", parentFqnHash + ".%"); + filter.queryParams.put("fqnHashNestedLevel", parentFqnHash + ".%.%"); + + condition += + " AND fqnHash LIKE :fqnHashSingleLevel AND fqnHash NOT LIKE :fqnHashNestedLevel"; + } else if (Boolean.TRUE.toString().equals(hierarchyFilter)) { + // For hierarchy API, when directChildrenOf is null, show only root domains + condition += + " AND NOT EXISTS (SELECT 1 FROM entity_relationship er WHERE er.toId = domain_entity.id AND er.fromEntity = 'domain' AND er.toEntity = 'domain' AND er.relation = " + + Relationship.CONTAINS.ordinal() + + ")"; + } + + if (!nullOrEmpty(offsetParam) && Integer.parseInt(offsetParam) >= 0) { + return listAfter( + getTableName(), + filter.getQueryParams(), + condition, + limit, + Integer.parseInt(offsetParam)); + } + + return listAfter( + getTableName(), filter.getQueryParams(), condition, limit, afterName, afterId); + } + + @SqlQuery("SELECT json FROM domain_entity WHERE fqnHash LIKE :concatFqnhash ") + List getNestedDomains( + @BindConcat( + value = "concatFqnhash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash); + + @SqlQuery("SELECT COUNT(*) FROM domain_entity WHERE fqnHash LIKE :concatFqnhash ") + int countNestedDomains( + @BindConcat( + value = "concatFqnhash", + parts = {":fqnhash", ".%"}, + hash = true) + String fqnhash); + } + + interface DataProductDAO extends EntityDAO { + @Override + default String getTableName() { + return "data_product_entity"; + } + + @Override + default Class getEntityClass() { + return DataProduct.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + } + + interface DataContractDAO extends EntityDAO { + @Override + default String getTableName() { + return "data_contract_entity"; + } + + @Override + default Class getEntityClass() { + return DataContract.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM data_contract_entity WHERE JSON_EXTRACT(json, '$.entity.id') = :entityId AND JSON_EXTRACT(json, '$.entity.type') = :entityType AND (JSON_EXTRACT(json, '$.deleted') IS NULL OR JSON_EXTRACT(json, '$.deleted') = false) LIMIT 1", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM data_contract_entity WHERE json#>>'{entity,id}' = :entityId AND json#>>'{entity,type}' = :entityType AND (json->>'deleted' IS NULL OR json->>'deleted' = 'false') LIMIT 1", + connectionType = POSTGRES) + String getContractByEntityId( + @Bind("entityId") String entityId, @Bind("entityType") String entityType); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/InheritanceParentCache.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/InheritanceParentCache.java new file mode 100644 index 000000000000..ddb303eedc44 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/InheritanceParentCache.java @@ -0,0 +1,151 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.Include.ALL; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; +import org.openmetadata.service.Entity; + +/** Thread-local parent caches for entity inheritance resolution, extracted from + * EntityRepository. Standalone (no repository state); EntityRepository delegates. */ +public final class InheritanceParentCache { + + private final ThreadLocal> parentCacheForPrepare = new ThreadLocal<>(); + private static final ThreadLocal> + inheritanceParentCache = ThreadLocal.withInitial(HashMap::new); + + record InheritanceCacheKey(String entityType, UUID entityId, String fieldsKey) {} + + public EntityInterface getCachedParentOrLoad( + EntityReference ref, String fields, Include include) { + var cache = parentCacheForPrepare.get(); + if (cache != null && ref != null && ref.getId() != null) { + var cached = cache.get(ref.getId()); + if (cached != null) return cached; + } + return Entity.getEntity(ref, fields, include); + } + + /** Store preloaded parents in the thread-local cache. */ + public void setParentCache(Map cache) { + parentCacheForPrepare.set(cache); + } + + /** Clear the parent cache after bulk prepare. */ + public void clearParentCache() { + parentCacheForPrepare.remove(); + inheritanceParentCache.remove(); + } + + public static void clearInheritanceParentCache() { + inheritanceParentCache.remove(); + } + + public EntityInterface getCachedInheritanceParent(EntityReference parentRef, String fields) { + if (parentRef == null || parentRef.getId() == null || nullOrEmpty(parentRef.getType())) { + return null; + } + Map cache = inheritanceParentCache.get(); + InheritanceCacheKey directKey = inheritanceCacheKey(parentRef, fields); + EntityInterface direct = cache.get(directKey); + if (direct != null) { + return direct; + } + + // Reuse a superset entry when the same parent was already loaded with broader fields + // (for example "owners,domains,retentionPeriod" can serve "owners,domains"). + Set requestedFields = parseFieldSet(directKey.fieldsKey()); + for (Entry entry : cache.entrySet()) { + InheritanceCacheKey cachedKey = entry.getKey(); + if (!cachedKey.entityType().equals(parentRef.getType()) + || !cachedKey.entityId().equals(parentRef.getId())) { + continue; + } + if (parseFieldSet(cachedKey.fieldsKey()).containsAll(requestedFields)) { + return entry.getValue(); + } + } + return null; + } + + public

P getOrLoadInheritanceParent( + EntityReference parentRef, String fields, Class

parentClass) { + if (parentRef == null || parentRef.getId() == null || nullOrEmpty(parentRef.getType())) { + return null; + } + EntityInterface parent = getCachedInheritanceParent(parentRef, fields); + if (parent == null) { + parent = Entity.getEntityForInheritance(parentRef.getType(), parentRef.getId(), fields, ALL); + cacheInheritanceParent(parentRef, fields, parent); + } + if (!parentClass.isInstance(parent)) { + return null; + } + return parentClass.cast(parent); + } + + public void cacheInheritanceParent( + EntityReference parentRef, String fields, EntityInterface parent) { + if (parentRef == null || parentRef.getId() == null || nullOrEmpty(parentRef.getType())) { + return; + } + if (parent == null || parent.getId() == null) { + return; + } + inheritanceParentCache.get().put(inheritanceCacheKey(parentRef, fields), parent); + } + + private InheritanceCacheKey inheritanceCacheKey(EntityReference parentRef, String fields) { + return new InheritanceCacheKey( + parentRef.getType(), parentRef.getId(), normalizeFieldList(fields)); + } + + private String normalizeFieldList(String fields) { + if (fields == null || fields.isBlank()) { + return ""; + } + // Canonicalize field order so cache keys are stable across equivalent requests + // (e.g. "owners,domains" and "domains, owners" should share the same parent entry). + return fields + .lines() + .flatMap(line -> Stream.of(line.split(","))) + .map(String::trim) + .filter(field -> !field.isEmpty()) + .distinct() + .sorted() + .collect(Collectors.joining(",")); + } + + private Set parseFieldSet(String fields) { + if (fields == null || fields.isBlank()) { + return Collections.emptySet(); + } + return Stream.of(fields.split(",")) + .map(String::trim) + .filter(field -> !field.isEmpty()) + .collect(Collectors.toSet()); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KnowledgeAssetDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KnowledgeAssetDAOs.java new file mode 100644 index 000000000000..4d04088b8fe7 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/KnowledgeAssetDAOs.java @@ -0,0 +1,382 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.Relationship.HAS; +import static org.openmetadata.schema.type.Relationship.OWNS; +import static org.openmetadata.service.Entity.TEAM; +import static org.openmetadata.service.Entity.USER; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindMap; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.jdbi.BindFQN; + +public interface KnowledgeAssetDAOs { + @CreateSqlObject + FolderDAO folderDAO(); + + @CreateSqlObject + ContextFileDAO contextFileDAO(); + + @CreateSqlObject + ContextFileContentDAO contextFileContentDAO(); + + @CreateSqlObject + KnowledgePageDAO knowledgePageDAO(); + + interface FolderDAO extends EntityDAO { + @Override + default String getTableName() { + return "drive_folder"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.data.Folder.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + } + + interface ContextFileDAO extends EntityDAO { + @Override + default String getTableName() { + return "context_file"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.data.ContextFile.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + } + + interface ContextFileContentDAO + extends EntityDAO { + @Override + default String getTableName() { + return "context_file_content"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.data.ContextFileContent.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM context_file_content " + + "WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.contextFile.id')) = :contextFileId", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM context_file_content " + + "WHERE json->'contextFile'->>'id' = :contextFileId", + connectionType = POSTGRES) + List listByContextFileId(@Bind("contextFileId") String contextFileId); + } + + interface KnowledgePageDAO extends EntityDAO { + String KNOWLEDGE_PAGE_ENTITY = "page"; + + @Override + default String getTableName() { + return "knowledge_center"; + } + + @Override + default Class getEntityClass() { + return org.openmetadata.schema.entity.data.Page.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + + /** + * When the caller supplies {@code entityId} + {@code entityType} (e.g. from a data-asset + * page that wants the list of knowledge pages referencing it), join against + * {@code entity_relationship} so that only pages whose {@code relatedEntities} contains + * the target entity are returned. Without this override, the base {@code EntityDAO.listAfter} + * ignores those params and returns every knowledge page — breaking the Knowledge + * Articles right-panel widget (and the corresponding playwright assertions). + */ + @Override + default int listCount(ListFilter filter) { + String entityId = filter.getQueryParam("entityId"); + String entityType = filter.getQueryParam("entityType"); + String knowledgePageType = filter.getQueryParam("pageType"); + String tagFQN = filter.getQueryParam("tagFQN"); + String tagListCondition = + "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; + String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; + if (nullOrEmpty(tagFQN)) { + tagListCondition = ""; + tagFilterCondition = "WHERE"; + } + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { + String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); + String condition = + String.format( + "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s " + + "entity_relationship.fromId IN (%s) %s" + + "and entity_relationship.toEntity = :toEntityType %s", + tagListCondition, + tagFilterCondition, + entityId, + getRelationCondition(entityType), + knowledgePageTypeQuery); + bindMap.put("toEntityType", KNOWLEDGE_PAGE_ENTITY); + bindMap.put("tagFQN", tagFQN); + if (!nullOrEmpty(knowledgePageTypeQuery)) { + bindMap.put("pageType", knowledgePageType); + } + return listKnowledgePageCountByEntity(condition, bindMap); + } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) + || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { + throw new IllegalArgumentException( + "Query Param Entity Id and Entity Type both needs to be provided."); + } + + String knowledgePageQueryClause = + String.format( + "%s %s %s", + tagListCondition, + tagFilterCondition, + getKnowledgePageTypeQuery("", knowledgePageType)); + return listCount( + getTableName(), + getNameHashColumn(), + filter.getQueryParams(), + getKnowledgePageWhereClause(knowledgePageQueryClause)); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String entityId = filter.getQueryParam("entityId"); + String entityType = filter.getQueryParam("entityType"); + String knowledgePageType = filter.getQueryParam("pageType"); + String tagFQN = filter.getQueryParam("tagFQN"); + String tagListCondition = + "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; + String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; + if (nullOrEmpty(tagFQN)) { + tagListCondition = ""; + tagFilterCondition = "WHERE"; + } + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { + String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); + String condition = + String.format( + "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s entity_relationship.fromId IN (%s) " + + "%s and entity_relationship.toEntity = :toEntity %s " + + "and (knowledge_center.name < :beforeName OR (knowledge_center.name = :beforeName AND knowledge_center.id < :beforeId)) order by knowledge_center.name DESC,knowledge_center.id DESC LIMIT :limit", + tagListCondition, + tagFilterCondition, + entityId, + getRelationCondition(entityType), + knowledgePageTypeQuery); + bindMap.put("toEntity", KNOWLEDGE_PAGE_ENTITY); + bindMap.put("beforeName", beforeName); + bindMap.put("beforeId", beforeId); + bindMap.put("limit", limit); + bindMap.put("tagFQN", tagFQN); + if (!nullOrEmpty(knowledgePageTypeQuery)) { + bindMap.put("pageType", knowledgePageType); + } + return listBeforeKnowledgePageByEntityId(condition, bindMap); + } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) + || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { + throw new IllegalArgumentException( + "Query Param Entity Id and Entity Type both needs to be provided."); + } + String knowledgePageQueryClause = + String.format( + "%s %s %s", + tagListCondition, + tagFilterCondition, + getKnowledgePageTypeQuery("", knowledgePageType)); + beforeName = FullyQualifiedName.unquoteName(beforeName); + return listBefore( + getTableName(), + filter.getQueryParams(), + getKnowledgePageWhereClause(knowledgePageQueryClause), + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String entityId = filter.getQueryParam("entityId"); + String entityType = filter.getQueryParam("entityType"); + String knowledgePageType = filter.getQueryParam("pageType"); + String tagFQN = filter.getQueryParam("tagFQN"); + String tagListCondition = + "INNER JOIN tag_usage ON knowledge_center.fqnHash = tag_usage.targetFQNHash"; + String tagFilterCondition = "WHERE tag_usage.tagFQN = :tagFQN and "; + if (nullOrEmpty(tagFQN)) { + tagListCondition = ""; + tagFilterCondition = "WHERE"; + } + Map bindMap = new HashMap<>(); + if (!nullOrEmpty(entityId) && !nullOrEmpty(entityType)) { + String knowledgePageTypeQuery = getKnowledgePageTypeQuery("AND", knowledgePageType); + String condition = + String.format( + "INNER JOIN entity_relationship ON knowledge_center.id = entity_relationship.toId %s %s entity_relationship.fromId IN (%s) " + + "%s and entity_relationship.toEntity = :toEntity %s " + + "and (knowledge_center.name > :afterName OR (knowledge_center.name = :afterName AND knowledge_center.id > :afterId)) order by knowledge_center.name ASC,knowledge_center.id ASC LIMIT :limit", + tagListCondition, + tagFilterCondition, + entityId, + getRelationCondition(entityType), + knowledgePageTypeQuery); + bindMap.put("toEntity", KNOWLEDGE_PAGE_ENTITY); + bindMap.put("afterName", afterName); + bindMap.put("afterId", afterId); + bindMap.put("limit", limit); + bindMap.put("tagFQN", tagFQN); + if (!nullOrEmpty(knowledgePageTypeQuery)) { + bindMap.put("pageType", knowledgePageType); + } + return listAfterKnowledgePageByEntityId(condition, bindMap); + } else if ((!nullOrEmpty(entityId) && nullOrEmpty(entityType)) + || (nullOrEmpty(entityId) && !nullOrEmpty(entityType))) { + throw new IllegalArgumentException( + "Query Param Entity Id and Entity Type both needs to be provided."); + } + String knowledgePageQueryClause = + String.format( + "%s %s %s", + tagListCondition, + tagFilterCondition, + getKnowledgePageTypeQuery("", knowledgePageType)); + afterName = FullyQualifiedName.unquoteName(afterName); + return listAfter( + getTableName(), + filter.getQueryParams(), + getKnowledgePageWhereClause(knowledgePageQueryClause), + limit, + afterName, + afterId); + } + + private String getRelationCondition(String entityType) { + // Users/teams "own" pages (membership-based); every other entity type reaches the page + // through a HAS relationship (the page's relatedEntities list). + String owns = String.valueOf(OWNS.ordinal()); + String has = String.valueOf(HAS.ordinal()); + if (entityType.equals(USER) || entityType.equals(TEAM)) { + return String.format(" and entity_relationship.relation = %s ", owns); + } else { + return String.format(" and entity_relationship.relation = %s ", has); + } + } + + private String getKnowledgePageWhereClause(String knowledgePageQueryClause) { + return nullOrEmpty(knowledgePageQueryClause) ? "WHERE TRUE" : knowledgePageQueryClause; + } + + private String getKnowledgePageTypeQuery(String clause, String type) { + if (!nullOrEmpty(type)) { + if (Boolean.TRUE.equals( + org.openmetadata.service.resources.databases.DatasourceConfig.getInstance() + .isMySQL())) { + return String.format( + " %s JSON_EXTRACT(knowledge_center.json, '$.pageType') = :pageType", clause); + } else { + return String.format(" %s knowledge_center.json->>'pageType' = :pageType", clause); + } + } + if ("AND".equals(clause)) { + return ""; + } + return "TRUE"; + } + + @SqlQuery("SELECT knowledge_center.json FROM knowledge_center ") + List listAfterKnowledgePageByEntityId( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery( + "SELECT json FROM (SELECT knowledge_center.name,knowledge_center.id, knowledge_center.json FROM knowledge_center ) last_rows_subquery ORDER BY name,id") + List listBeforeKnowledgePageByEntityId( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery("SELECT count(*) FROM knowledge_center ") + int listKnowledgePageCountByEntity( + @Define("cond") String cond, @BindMap Map bindings); + + @SqlQuery( + "SELECT json " + + "FROM knowledge_center " + + "WHERE id NOT IN (" + + " SELECT toId FROM entity_relationship WHERE (relation = 0 AND toEntity = 'page') OR (relation = 9 AND toEntity = 'page')" + + ")") + List listTopLevelPages(); + + @SqlQuery( + "SELECT kc.json " + + "FROM knowledge_center kc " + + "JOIN entity_relationship er ON kc.id = er.toId " + + "WHERE er.fromId = :parentId " + + "AND (er.relation = 9 or er.relation = 0) " + + "AND er.toEntity = 'page'") + List listChildren(@Bind("parentId") String parentId); + + @ConnectionAwareSqlUpdate( + value = "UPDATE knowledge_center SET json = :json, fqnHash = :fqnHash WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE knowledge_center SET json = :json::jsonb, fqnHash = :fqnHash WHERE id = :id", + connectionType = POSTGRES) + void updateFullyQualifiedName( + @Bind("id") String pageId, @Bind("json") String json, @BindFQN("fqnHash") String fqnHash); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index cd01067f510e..9199f80eff4f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -100,7 +100,7 @@ import org.openmetadata.search.IndexMapping; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.rdf.RdfUpdater; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchIndexRetryQueue; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/OAuthDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/OAuthDAOs.java new file mode 100644 index 000000000000..f08dc0e6c319 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/OAuthDAOs.java @@ -0,0 +1,251 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.jdbi3.oauth.OAuthRecords; + +public interface OAuthDAOs { + @CreateSqlObject + OAuthClientDAO oauthClientDAO(); + + @CreateSqlObject + OAuthAuthorizationCodeDAO oauthAuthorizationCodeDAO(); + + @CreateSqlObject + OAuthAccessTokenDAO oauthAccessTokenDAO(); + + @CreateSqlObject + OAuthRefreshTokenDAO oauthRefreshTokenDAO(); + + @CreateSqlObject + McpPendingAuthRequestDAO mcpPendingAuthRequestDAO(); + + interface OAuthClientDAO { + @SqlQuery( + "SELECT id, client_id, client_secret_encrypted, client_name, redirect_uris, grant_types, token_endpoint_auth_method, scopes FROM oauth_clients WHERE client_id = :clientId") + @RegisterRowMapper(SystemTokenDAOs.OAuthClientRowMapper.class) + OAuthRecords.OAuthClientRecord findByClientId(@Bind("clientId") String clientId); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO oauth_clients (client_id, client_secret_encrypted, client_name, redirect_uris, grant_types, token_endpoint_auth_method, scopes) VALUES (:clientId, :clientSecret, :clientName, :redirectUris ::jsonb, :grantTypes ::jsonb, :authMethod, :scopes ::jsonb)", + connectionType = POSTGRES) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO oauth_clients (client_id, client_secret_encrypted, client_name, redirect_uris, grant_types, token_endpoint_auth_method, scopes) VALUES (:clientId, :clientSecret, :clientName, :redirectUris, :grantTypes, :authMethod, :scopes)", + connectionType = MYSQL) + void insert( + @Bind("clientId") String clientId, + @Bind("clientSecret") String clientSecret, + @Bind("clientName") String clientName, + @Bind("redirectUris") String redirectUris, + @Bind("grantTypes") String grantTypes, + @Bind("authMethod") String authMethod, + @Bind("scopes") String scopes); + + @SqlUpdate("DELETE FROM oauth_clients WHERE client_id = :clientId") + void delete(@Bind("clientId") String clientId); + } + + interface OAuthAuthorizationCodeDAO { + @SqlQuery( + "SELECT code, client_id, user_name, code_challenge, code_challenge_method, redirect_uri, scopes, expires_at, used FROM oauth_authorization_codes WHERE code = :code") + @RegisterRowMapper(SystemTokenDAOs.OAuthAuthorizationCodeRowMapper.class) + OAuthRecords.OAuthAuthorizationCodeRecord findByCode(@Bind("code") String code); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO oauth_authorization_codes (code, client_id, user_name, code_challenge, code_challenge_method, redirect_uri, scopes, expires_at) VALUES (:code, :clientId, :userName, :codeChallenge, :codeChallengeMethod, :redirectUri, :scopes ::jsonb, :expiresAt)", + connectionType = POSTGRES) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO oauth_authorization_codes (code, client_id, user_name, code_challenge, code_challenge_method, redirect_uri, scopes, expires_at) VALUES (:code, :clientId, :userName, :codeChallenge, :codeChallengeMethod, :redirectUri, :scopes, :expiresAt)", + connectionType = MYSQL) + void insert( + @Bind("code") String code, + @Bind("clientId") String clientId, + @Bind("userName") String userName, + @Bind("codeChallenge") String codeChallenge, + @Bind("codeChallengeMethod") String codeChallengeMethod, + @Bind("redirectUri") String redirectUri, + @Bind("scopes") String scopes, + @Bind("expiresAt") long expiresAt); + + @SqlUpdate( + "UPDATE oauth_authorization_codes SET used = TRUE WHERE code = :code AND used = FALSE") + int markAsUsedAtomic(@Bind("code") String code); + + @SqlUpdate("DELETE FROM oauth_authorization_codes WHERE code = :code") + void delete(@Bind("code") String code); + + @SqlUpdate("DELETE FROM oauth_authorization_codes WHERE expires_at < :currentTime") + void deleteExpired(@Bind("currentTime") long currentTime); + } + + interface OAuthAccessTokenDAO { + @SqlQuery( + "SELECT id, token_hash, access_token_encrypted, client_id, user_name, scopes, expires_at FROM oauth_access_tokens WHERE token_hash = :tokenHash") + @RegisterRowMapper(SystemTokenDAOs.OAuthAccessTokenRowMapper.class) + OAuthRecords.OAuthAccessTokenRecord findByTokenHash(@Bind("tokenHash") String tokenHash); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO oauth_access_tokens (token_hash, access_token_encrypted, client_id, user_name, scopes, expires_at) VALUES (:tokenHash, :accessTokenEncrypted, :clientId, :userName, :scopes ::jsonb, :expiresAt)", + connectionType = POSTGRES) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO oauth_access_tokens (token_hash, access_token_encrypted, client_id, user_name, scopes, expires_at) VALUES (:tokenHash, :accessTokenEncrypted, :clientId, :userName, :scopes, :expiresAt)", + connectionType = MYSQL) + void insert( + @Bind("tokenHash") String tokenHash, + @Bind("accessTokenEncrypted") String accessTokenEncrypted, + @Bind("clientId") String clientId, + @Bind("userName") String userName, + @Bind("scopes") String scopes, + @Bind("expiresAt") long expiresAt); + + @SqlUpdate("DELETE FROM oauth_access_tokens WHERE token_hash = :tokenHash") + void delete(@Bind("tokenHash") String tokenHash); + + @SqlUpdate("DELETE FROM oauth_access_tokens WHERE expires_at < :currentTime") + void deleteExpired(@Bind("currentTime") long currentTime); + } + + interface OAuthRefreshTokenDAO { + @SqlQuery( + "SELECT id, token_hash, refresh_token_encrypted, client_id, user_name, scopes, expires_at, revoked FROM oauth_refresh_tokens WHERE token_hash = :tokenHash") + @RegisterRowMapper(SystemTokenDAOs.OAuthRefreshTokenRowMapper.class) + OAuthRecords.OAuthRefreshTokenRecord findByTokenHash(@Bind("tokenHash") String tokenHash); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO oauth_refresh_tokens (token_hash, refresh_token_encrypted, client_id, user_name, scopes, expires_at) VALUES (:tokenHash, :refreshTokenEncrypted, :clientId, :userName, :scopes ::jsonb, :expiresAt)", + connectionType = POSTGRES) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO oauth_refresh_tokens (token_hash, refresh_token_encrypted, client_id, user_name, scopes, expires_at) VALUES (:tokenHash, :refreshTokenEncrypted, :clientId, :userName, :scopes, :expiresAt)", + connectionType = MYSQL) + void insert( + @Bind("tokenHash") String tokenHash, + @Bind("refreshTokenEncrypted") String refreshTokenEncrypted, + @Bind("clientId") String clientId, + @Bind("userName") String userName, + @Bind("scopes") String scopes, + @Bind("expiresAt") long expiresAt); + + @SqlUpdate("UPDATE oauth_refresh_tokens SET revoked = TRUE WHERE token_hash = :tokenHash") + void revoke(@Bind("tokenHash") String tokenHash); + + @SqlUpdate( + "UPDATE oauth_refresh_tokens SET revoked = TRUE WHERE token_hash = :tokenHash AND revoked = FALSE") + int revokeAtomic(@Bind("tokenHash") String tokenHash); + + @SqlUpdate("DELETE FROM oauth_refresh_tokens WHERE token_hash = :tokenHash") + void delete(@Bind("tokenHash") String tokenHash); + + @SqlUpdate("DELETE FROM oauth_refresh_tokens WHERE expires_at < :currentTime") + void deleteExpired(@Bind("currentTime") long currentTime); + + @SqlUpdate( + "UPDATE oauth_refresh_tokens SET revoked = TRUE WHERE client_id = :clientId AND user_name = :userName AND revoked = FALSE") + void revokeAllForUser(@Bind("clientId") String clientId, @Bind("userName") String userName); + } + + interface McpPendingAuthRequestDAO { + @SqlQuery( + "SELECT auth_request_id, client_id, code_challenge, code_challenge_method, redirect_uri, mcp_state, scopes, pac4j_state, pac4j_nonce, pac4j_code_verifier, expires_at FROM mcp_pending_auth_requests WHERE auth_request_id = :authRequestId") + @RegisterRowMapper(McpPendingAuthRequestRowMapper.class) + OAuthRecords.McpPendingAuthRequest findByAuthRequestId( + @Bind("authRequestId") String authRequestId); + + @SqlQuery( + "SELECT auth_request_id, client_id, code_challenge, code_challenge_method, redirect_uri, mcp_state, scopes, pac4j_state, pac4j_nonce, pac4j_code_verifier, expires_at FROM mcp_pending_auth_requests WHERE pac4j_state = :pac4jState") + @RegisterRowMapper(McpPendingAuthRequestRowMapper.class) + OAuthRecords.McpPendingAuthRequest findByPac4jState(@Bind("pac4jState") String pac4jState); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO mcp_pending_auth_requests (auth_request_id, client_id, code_challenge, code_challenge_method, redirect_uri, mcp_state, scopes, pac4j_state, pac4j_nonce, pac4j_code_verifier, expires_at) VALUES (:authRequestId, :clientId, :codeChallenge, :codeChallengeMethod, :redirectUri, :mcpState, :scopes ::jsonb, :pac4jState, :pac4jNonce, :pac4jCodeVerifier, :expiresAt)", + connectionType = POSTGRES) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO mcp_pending_auth_requests (auth_request_id, client_id, code_challenge, code_challenge_method, redirect_uri, mcp_state, scopes, pac4j_state, pac4j_nonce, pac4j_code_verifier, expires_at) VALUES (:authRequestId, :clientId, :codeChallenge, :codeChallengeMethod, :redirectUri, :mcpState, :scopes, :pac4jState, :pac4jNonce, :pac4jCodeVerifier, :expiresAt)", + connectionType = MYSQL) + void insert( + @Bind("authRequestId") String authRequestId, + @Bind("clientId") String clientId, + @Bind("codeChallenge") String codeChallenge, + @Bind("codeChallengeMethod") String codeChallengeMethod, + @Bind("redirectUri") String redirectUri, + @Bind("mcpState") String mcpState, + @Bind("scopes") String scopes, + @Bind("pac4jState") String pac4jState, + @Bind("pac4jNonce") String pac4jNonce, + @Bind("pac4jCodeVerifier") String pac4jCodeVerifier, + @Bind("expiresAt") long expiresAt); + + @SqlUpdate( + "UPDATE mcp_pending_auth_requests SET pac4j_state = :pac4jState, pac4j_nonce = :pac4jNonce, pac4j_code_verifier = :pac4jCodeVerifier WHERE auth_request_id = :authRequestId") + void updatePac4jSession( + @Bind("authRequestId") String authRequestId, + @Bind("pac4jState") String pac4jState, + @Bind("pac4jNonce") String pac4jNonce, + @Bind("pac4jCodeVerifier") String pac4jCodeVerifier); + + @SqlUpdate("DELETE FROM mcp_pending_auth_requests WHERE auth_request_id = :authRequestId") + void delete(@Bind("authRequestId") String authRequestId); + + @SqlUpdate("DELETE FROM mcp_pending_auth_requests WHERE expires_at < :currentTime") + void deleteExpired(@Bind("currentTime") long currentTime); + } + + class McpPendingAuthRequestRowMapper implements RowMapper { + @Override + public OAuthRecords.McpPendingAuthRequest map(ResultSet rs, StatementContext ctx) + throws SQLException { + String scopesJson = rs.getString("scopes"); + List scopes = + scopesJson != null + ? org.openmetadata.schema.utils.JsonUtils.readValue( + scopesJson, new com.fasterxml.jackson.core.type.TypeReference>() {}) + : List.of(); + return new OAuthRecords.McpPendingAuthRequest( + rs.getString("auth_request_id"), + rs.getString("client_id"), + rs.getString("code_challenge"), + rs.getString("code_challenge_method"), + rs.getString("redirect_uri"), + rs.getString("mcp_state"), + scopes, + rs.getString("pac4j_state"), + rs.getString("pac4j_nonce"), + rs.getString("pac4j_code_verifier"), + rs.getLong("expires_at")); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PersonaRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PersonaRepository.java index c230ac393a7e..7c0f215efc94 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PersonaRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PersonaRepository.java @@ -111,13 +111,14 @@ private List getUsers(Persona persona) { private void unsetExistingDefaultPersona(String newDefaultPersonaId) { // Capture both id and FQN *before* the bulk update. The bulk update rewrites JSON directly — // bypassing invalidateCachesAfterStore — so every affected persona would keep stale - // "default=true" in both CACHE_WITH_ID and CACHE_WITH_NAME variants. Passing fqn lets + // "default=true" in both EntityCaches.CACHE_WITH_ID and EntityCaches.CACHE_WITH_NAME variants. + // Passing fqn lets // invalidateCacheForEntity drop the by-name cache alongside the by-id one. List affected = daoCollection.personaDAO().findOtherDefaultPersonaIdsWithFqn(newDefaultPersonaId); daoCollection.personaDAO().unsetOtherDefaultPersonas(newDefaultPersonaId); for (EntityDAO.EntityIdFqnPair persona : affected) { - invalidateCacheForEntity(Entity.PERSONA, persona.id, persona.fqn); + EntityCacheInvalidator.invalidateCacheForEntity(Entity.PERSONA, persona.id, persona.fqn); } } @@ -154,13 +155,16 @@ protected void preDelete(Persona persona, String deletedBy) { // Users/teams that had this persona cached embed the persona reference in their serialized // JSON. Drop their cached entries so the next read rebuilds without the now-deleted persona. for (EntityReference user : listOrEmpty(users)) { - invalidateCacheForEntity(USER, user.getId(), user.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + USER, user.getId(), user.getFullyQualifiedName()); } for (EntityReference user : listOrEmpty(defaultUsers)) { - invalidateCacheForEntity(USER, user.getId(), user.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + USER, user.getId(), user.getFullyQualifiedName()); } for (EntityReference team : listOrEmpty(teams)) { - invalidateCacheForEntity(Entity.TEAM, team.getId(), team.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + Entity.TEAM, team.getId(), team.getFullyQualifiedName()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/RdfInfraDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/RdfInfraDAOs.java new file mode 100644 index 000000000000..70f4c82886ea --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/RdfInfraDAOs.java @@ -0,0 +1,766 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; + +public interface RdfInfraDAOs { + @CreateSqlObject + RdfIndexJobDAO rdfIndexJobDAO(); + + @CreateSqlObject + RdfIndexPartitionDAO rdfIndexPartitionDAO(); + + @CreateSqlObject + RdfReindexLockDAO rdfReindexLockDAO(); + + @CreateSqlObject + RdfIndexServerStatsDAO rdfIndexServerStatsDAO(); + + /** DAO for distributed RDF index jobs. */ + interface RdfIndexJobDAO { + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO rdf_index_job (id, status, jobConfiguration, totalRecords, processedRecords, " + + "successRecords, failedRecords, stats, createdBy, createdAt, updatedAt) " + + "VALUES (:id, :status, :jobConfiguration, :totalRecords, :processedRecords, " + + ":successRecords, :failedRecords, :stats, :createdBy, :createdAt, :updatedAt)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO rdf_index_job (id, status, jobConfiguration, totalRecords, processedRecords, " + + "successRecords, failedRecords, stats, createdBy, createdAt, updatedAt) " + + "VALUES (:id, :status, :jobConfiguration::jsonb, :totalRecords, :processedRecords, " + + ":successRecords, :failedRecords, :stats::jsonb, :createdBy, :createdAt, :updatedAt)", + connectionType = POSTGRES) + void insert( + @Bind("id") String id, + @Bind("status") String status, + @Bind("jobConfiguration") String jobConfiguration, + @Bind("totalRecords") long totalRecords, + @Bind("processedRecords") long processedRecords, + @Bind("successRecords") long successRecords, + @Bind("failedRecords") long failedRecords, + @Bind("stats") String stats, + @Bind("createdBy") String createdBy, + @Bind("createdAt") long createdAt, + @Bind("updatedAt") long updatedAt); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE rdf_index_job SET status = :status, processedRecords = :processedRecords, " + + "successRecords = :successRecords, failedRecords = :failedRecords, stats = :stats, " + + "startedAt = :startedAt, completedAt = :completedAt, updatedAt = :updatedAt, " + + "errorMessage = :errorMessage WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE rdf_index_job SET status = :status, processedRecords = :processedRecords, " + + "successRecords = :successRecords, failedRecords = :failedRecords, stats = :stats::jsonb, " + + "startedAt = :startedAt, completedAt = :completedAt, updatedAt = :updatedAt, " + + "errorMessage = :errorMessage WHERE id = :id", + connectionType = POSTGRES) + void update( + @Bind("id") String id, + @Bind("status") String status, + @Bind("processedRecords") long processedRecords, + @Bind("successRecords") long successRecords, + @Bind("failedRecords") long failedRecords, + @Bind("stats") String stats, + @Bind("startedAt") Long startedAt, + @Bind("completedAt") Long completedAt, + @Bind("updatedAt") long updatedAt, + @Bind("errorMessage") String errorMessage); + + @SqlUpdate("UPDATE rdf_index_job SET updatedAt = :updatedAt WHERE id = :id") + void touchJob(@Bind("id") String id, @Bind("updatedAt") long updatedAt); + + @SqlQuery("SELECT * FROM rdf_index_job WHERE id = :id") + @RegisterRowMapper(RdfIndexJobMapper.class) + RdfIndexJobRecord findById(@Bind("id") String id); + + @SqlQuery("SELECT * FROM rdf_index_job WHERE status IN () ORDER BY createdAt DESC") + @RegisterRowMapper(RdfIndexJobMapper.class) + List findByStatuses(@BindList("statuses") List statuses); + + @SqlQuery( + "SELECT * FROM rdf_index_job WHERE status IN () ORDER BY createdAt DESC LIMIT :limit") + @RegisterRowMapper(RdfIndexJobMapper.class) + List findByStatusesWithLimit( + @BindList("statuses") List statuses, @Bind("limit") int limit); + + @SqlQuery("SELECT id FROM rdf_index_job WHERE status IN ('READY', 'RUNNING', 'STOPPING')") + List getRunningJobIds(); + + @SqlUpdate("DELETE FROM rdf_index_job") + void deleteAll(); + + class RdfIndexJobMapper implements RowMapper { + @Override + public RdfIndexJobRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new RdfIndexJobRecord( + rs.getString("id"), + rs.getString("status"), + rs.getString("jobConfiguration"), + rs.getLong("totalRecords"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getString("stats"), + rs.getString("createdBy"), + rs.getLong("createdAt"), + (Long) rs.getObject("startedAt"), + (Long) rs.getObject("completedAt"), + rs.getLong("updatedAt"), + rs.getString("errorMessage")); + } + } + + record RdfIndexJobRecord( + String id, + String status, + String jobConfiguration, + long totalRecords, + long processedRecords, + long successRecords, + long failedRecords, + String stats, + String createdBy, + long createdAt, + Long startedAt, + Long completedAt, + long updatedAt, + String errorMessage) {} + } + + /** DAO for distributed RDF partitions. */ + interface RdfIndexPartitionDAO { + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO rdf_index_partition (id, jobId, entityType, partitionIndex, rangeStart, rangeEnd, " + + "estimatedCount, workUnits, priority, status, processingCursor, claimableAt) " + + "VALUES (:id, :jobId, :entityType, :partitionIndex, :rangeStart, :rangeEnd, " + + ":estimatedCount, :workUnits, :priority, :status, :cursor, :claimableAt)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO rdf_index_partition (id, jobId, entityType, partitionIndex, rangeStart, rangeEnd, " + + "estimatedCount, workUnits, priority, status, processingCursor, claimableAt) " + + "VALUES (:id, :jobId, :entityType, :partitionIndex, :rangeStart, :rangeEnd, " + + ":estimatedCount, :workUnits, :priority, :status, :cursor, :claimableAt)", + connectionType = POSTGRES) + void insert( + @Bind("id") String id, + @Bind("jobId") String jobId, + @Bind("entityType") String entityType, + @Bind("partitionIndex") int partitionIndex, + @Bind("rangeStart") long rangeStart, + @Bind("rangeEnd") long rangeEnd, + @Bind("estimatedCount") long estimatedCount, + @Bind("workUnits") long workUnits, + @Bind("priority") int priority, + @Bind("status") String status, + @Bind("cursor") long cursor, + @Bind("claimableAt") long claimableAt); + + @SqlUpdate( + "UPDATE rdf_index_partition SET status = :status, processingCursor = :cursor, " + + "processedCount = :processedCount, successCount = :successCount, failedCount = :failedCount, " + + "assignedServer = :assignedServer, claimedAt = :claimedAt, startedAt = :startedAt, " + + "completedAt = :completedAt, lastUpdateAt = :lastUpdateAt, lastError = :lastError, " + + "retryCount = :retryCount WHERE id = :id") + void update( + @Bind("id") String id, + @Bind("status") String status, + @Bind("cursor") long cursor, + @Bind("processedCount") long processedCount, + @Bind("successCount") long successCount, + @Bind("failedCount") long failedCount, + @Bind("assignedServer") String assignedServer, + @Bind("claimedAt") Long claimedAt, + @Bind("startedAt") Long startedAt, + @Bind("completedAt") Long completedAt, + @Bind("lastUpdateAt") Long lastUpdateAt, + @Bind("lastError") String lastError, + @Bind("retryCount") int retryCount); + + @SqlUpdate( + "UPDATE rdf_index_partition SET processingCursor = :cursor, processedCount = :processedCount, " + + "successCount = :successCount, failedCount = :failedCount, lastUpdateAt = :lastUpdateAt " + + "WHERE id = :id") + void updateProgress( + @Bind("id") String id, + @Bind("cursor") long cursor, + @Bind("processedCount") long processedCount, + @Bind("successCount") long successCount, + @Bind("failedCount") long failedCount, + @Bind("lastUpdateAt") long lastUpdateAt); + + @SqlUpdate("UPDATE rdf_index_partition SET lastUpdateAt = :lastUpdateAt WHERE id = :id") + void updateHeartbeat(@Bind("id") String id, @Bind("lastUpdateAt") long lastUpdateAt); + + @SqlQuery("SELECT * FROM rdf_index_partition WHERE id = :id") + @RegisterRowMapper(RdfIndexPartitionMapper.class) + RdfIndexPartitionRecord findById(@Bind("id") String id); + + @SqlQuery( + "SELECT * FROM rdf_index_partition WHERE jobId = :jobId ORDER BY priority DESC, entityType, partitionIndex") + @RegisterRowMapper(RdfIndexPartitionMapper.class) + List findByJobId(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT COUNT(*) FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PENDING'") + int countPendingPartitions(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT COUNT(*) FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PROCESSING'") + int countInFlightPartitions(@Bind("jobId") String jobId); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE rdf_index_partition p " + + "JOIN (SELECT id FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PENDING' " + + "AND claimableAt <= :now " + + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED) t ON p.id = t.id " + + "SET p.status = 'PROCESSING', p.assignedServer = :serverId, p.claimedAt = :now, " + + "p.startedAt = :now, p.lastUpdateAt = :now", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE rdf_index_partition SET status = 'PROCESSING', " + + "assignedServer = :serverId, claimedAt = :now, startedAt = :now, lastUpdateAt = :now " + + "WHERE id = (SELECT id FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PENDING' " + + "AND claimableAt <= :now " + + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED)", + connectionType = POSTGRES) + int claimNextPartitionAtomic( + @Bind("jobId") String jobId, @Bind("serverId") String serverId, @Bind("now") long now); + + @SqlQuery( + "SELECT * FROM rdf_index_partition WHERE jobId = :jobId AND status = 'PROCESSING' " + + "AND assignedServer = :serverId AND claimedAt = :claimedAt " + + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1") + @RegisterRowMapper(RdfIndexPartitionMapper.class) + RdfIndexPartitionRecord findLatestClaimedPartition( + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("claimedAt") long claimedAt); + + @SqlUpdate( + "UPDATE rdf_index_partition SET status = 'PENDING', assignedServer = NULL, claimedAt = NULL, " + + "retryCount = retryCount + 1, lastError = 'Reclaimed due to stale heartbeat' " + + "WHERE jobId = :jobId AND status = 'PROCESSING' AND lastUpdateAt < :staleThreshold " + + "AND retryCount < :maxRetries") + int reclaimStalePartitionsForRetry( + @Bind("jobId") String jobId, + @Bind("staleThreshold") long staleThreshold, + @Bind("maxRetries") int maxRetries); + + @SqlUpdate( + "UPDATE rdf_index_partition SET status = 'FAILED', " + + "lastError = 'Exceeded max retries after stale heartbeat', completedAt = :now " + + "WHERE jobId = :jobId AND status = 'PROCESSING' AND lastUpdateAt < :staleThreshold " + + "AND retryCount >= :maxRetries") + int failStalePartitionsExceedingRetries( + @Bind("jobId") String jobId, + @Bind("staleThreshold") long staleThreshold, + @Bind("maxRetries") int maxRetries, + @Bind("now") long now); + + @SqlUpdate( + "UPDATE rdf_index_partition SET status = 'CANCELLED' WHERE jobId = :jobId AND status = 'PENDING'") + int cancelPendingPartitions(@Bind("jobId") String jobId); + + @SqlUpdate( + "UPDATE rdf_index_partition SET status = 'CANCELLED', " + + "lastError = 'Stopped by user', completedAt = :now, lastUpdateAt = :now " + + "WHERE jobId = :jobId AND status IN ('PENDING','PROCESSING')") + int cancelInFlightPartitions(@Bind("jobId") String jobId, @Bind("now") long now); + + @SqlQuery( + "SELECT COUNT(*) FROM rdf_index_partition " + + "WHERE jobId = :jobId AND status = 'PROCESSING' AND assignedServer = :serverId") + int countInFlightPartitionsForServer( + @Bind("jobId") String jobId, @Bind("serverId") String serverId); + + @SqlQuery("SELECT COUNT(*) FROM rdf_index_partition WHERE jobId = :jobId AND status = :status") + int countPartitionsByStatus(@Bind("jobId") String jobId, @Bind("status") String status); + + /** + * Status-guarded variant of {@link #update}: only writes if the row is still + * PROCESSING. Workers use this on completion so that a concurrent Stop + * (which moves the row to CANCELLED) isn't overwritten back to + * COMPLETED/FAILED, which would make the Stop button look unreliable. + * Returns the number of rows updated (0 means the row was no longer + * PROCESSING and the caller should skip side effects like server-stat + * increments). + */ + @SqlUpdate( + "UPDATE rdf_index_partition SET status = :status, processingCursor = :cursor, " + + "processedCount = :processedCount, successCount = :successCount, failedCount = :failedCount, " + + "assignedServer = :assignedServer, claimedAt = :claimedAt, startedAt = :startedAt, " + + "completedAt = :completedAt, lastUpdateAt = :lastUpdateAt, lastError = :lastError, " + + "retryCount = :retryCount WHERE id = :id AND status = 'PROCESSING'") + int updateIfProcessing( + @Bind("id") String id, + @Bind("status") String status, + @Bind("cursor") long cursor, + @Bind("processedCount") long processedCount, + @Bind("successCount") long successCount, + @Bind("failedCount") long failedCount, + @Bind("assignedServer") String assignedServer, + @Bind("claimedAt") Long claimedAt, + @Bind("startedAt") Long startedAt, + @Bind("completedAt") Long completedAt, + @Bind("lastUpdateAt") Long lastUpdateAt, + @Bind("lastError") String lastError, + @Bind("retryCount") int retryCount); + + @SqlUpdate( + "UPDATE rdf_index_partition SET status = :status, assignedServer = NULL, claimedAt = NULL, " + + "lastError = :reason, lastUpdateAt = :updatedAt, completedAt = :completedAt " + + "WHERE jobId = :jobId AND status = 'PROCESSING' AND assignedServer = :serverId") + int releaseProcessingPartitions( + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("status") String status, + @Bind("reason") String reason, + @Bind("updatedAt") long updatedAt, + @Bind("completedAt") Long completedAt); + + @SqlQuery( + "SELECT entityType, " + + "SUM(estimatedCount) as totalRecords, " + + "SUM(processedCount) as processedRecords, " + + "SUM(successCount) as successRecords, " + + "SUM(failedCount) as failedRecords, " + + "COUNT(*) as totalPartitions, " + + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " + + "SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) as failedPartitions " + + "FROM rdf_index_partition WHERE jobId = :jobId GROUP BY entityType") + @RegisterRowMapper(RdfEntityStatsMapper.class) + List getEntityStats(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT " + + "SUM(estimatedCount) as totalRecords, " + + "SUM(processedCount) as processedRecords, " + + "SUM(successCount) as successRecords, " + + "SUM(failedCount) as failedRecords, " + + "COUNT(*) as totalPartitions, " + + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " + + "SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) as failedPartitions, " + + "SUM(CASE WHEN status = 'PENDING' THEN 1 ELSE 0 END) as pendingPartitions, " + + "SUM(CASE WHEN status = 'PROCESSING' THEN 1 ELSE 0 END) as processingPartitions " + + "FROM rdf_index_partition WHERE jobId = :jobId") + @RegisterRowMapper(RdfAggregatedStatsMapper.class) + RdfAggregatedStatsRecord getAggregatedStats(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT assignedServer, " + + "SUM(processedCount) as processedRecords, " + + "SUM(successCount) as successRecords, " + + "SUM(failedCount) as failedRecords, " + + "COUNT(*) as totalPartitions, " + + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " + + "SUM(CASE WHEN status = 'PROCESSING' THEN 1 ELSE 0 END) as processingPartitions " + + "FROM rdf_index_partition WHERE jobId = :jobId AND assignedServer IS NOT NULL " + + "GROUP BY assignedServer") + @RegisterRowMapper(RdfServerStatsMapper.class) + List getServerStats(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT DISTINCT assignedServer FROM rdf_index_partition " + + "WHERE jobId = :jobId AND assignedServer IS NOT NULL") + List getAssignedServers(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT lastError FROM rdf_index_partition " + + "WHERE jobId = :jobId AND lastError IS NOT NULL " + + "ORDER BY lastUpdateAt DESC LIMIT :limit") + List findRecentPartitionErrors(@Bind("jobId") String jobId, @Bind("limit") int limit); + + @SqlUpdate("DELETE FROM rdf_index_partition") + void deleteAll(); + + class RdfIndexPartitionMapper implements RowMapper { + @Override + public RdfIndexPartitionRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new RdfIndexPartitionRecord( + rs.getString("id"), + rs.getString("jobId"), + rs.getString("entityType"), + rs.getInt("partitionIndex"), + rs.getLong("rangeStart"), + rs.getLong("rangeEnd"), + rs.getLong("estimatedCount"), + rs.getLong("workUnits"), + rs.getInt("priority"), + rs.getString("status"), + rs.getLong("processingCursor"), + rs.getLong("processedCount"), + rs.getLong("successCount"), + rs.getLong("failedCount"), + rs.getString("assignedServer"), + (Long) rs.getObject("claimedAt"), + (Long) rs.getObject("startedAt"), + (Long) rs.getObject("completedAt"), + (Long) rs.getObject("lastUpdateAt"), + rs.getString("lastError"), + rs.getInt("retryCount"), + rs.getLong("claimableAt")); + } + } + + class RdfEntityStatsMapper implements RowMapper { + @Override + public RdfEntityStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new RdfEntityStatsRecord( + rs.getString("entityType"), + rs.getLong("totalRecords"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getInt("totalPartitions"), + rs.getInt("completedPartitions"), + rs.getInt("failedPartitions")); + } + } + + class RdfAggregatedStatsMapper implements RowMapper { + @Override + public RdfAggregatedStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new RdfAggregatedStatsRecord( + rs.getLong("totalRecords"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getInt("totalPartitions"), + rs.getInt("completedPartitions"), + rs.getInt("failedPartitions"), + rs.getInt("pendingPartitions"), + rs.getInt("processingPartitions")); + } + } + + class RdfServerStatsMapper implements RowMapper { + @Override + public RdfServerPartitionStatsRecord map(ResultSet rs, StatementContext ctx) + throws SQLException { + return new RdfServerPartitionStatsRecord( + rs.getString("assignedServer"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getInt("totalPartitions"), + rs.getInt("completedPartitions"), + rs.getInt("processingPartitions")); + } + } + + record RdfIndexPartitionRecord( + String id, + String jobId, + String entityType, + int partitionIndex, + long rangeStart, + long rangeEnd, + long estimatedCount, + long workUnits, + int priority, + String status, + long cursor, + long processedCount, + long successCount, + long failedCount, + String assignedServer, + Long claimedAt, + Long startedAt, + Long completedAt, + Long lastUpdateAt, + String lastError, + int retryCount, + long claimableAt) {} + + record RdfEntityStatsRecord( + String entityType, + long totalRecords, + long processedRecords, + long successRecords, + long failedRecords, + int totalPartitions, + int completedPartitions, + int failedPartitions) {} + + record RdfAggregatedStatsRecord( + long totalRecords, + long processedRecords, + long successRecords, + long failedRecords, + int totalPartitions, + int completedPartitions, + int failedPartitions, + int pendingPartitions, + int processingPartitions) {} + + record RdfServerPartitionStatsRecord( + String serverId, + long processedRecords, + long successRecords, + long failedRecords, + int totalPartitions, + int completedPartitions, + int processingPartitions) {} + } + + /** DAO for RDF distributed reindex lock. */ + interface RdfReindexLockDAO { + + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO rdf_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " + + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO rdf_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " + + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt) " + + "ON CONFLICT (lockKey) DO NOTHING", + connectionType = POSTGRES) + int insertIfNotExists( + @Bind("lockKey") String lockKey, + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("acquiredAt") long acquiredAt, + @Bind("lastHeartbeat") long lastHeartbeat, + @Bind("expiresAt") long expiresAt); + + @SqlUpdate( + "UPDATE rdf_reindex_lock SET lastHeartbeat = :lastHeartbeat, expiresAt = :expiresAt " + + "WHERE lockKey = :lockKey AND jobId = :jobId") + int updateHeartbeat( + @Bind("lockKey") String lockKey, + @Bind("jobId") String jobId, + @Bind("lastHeartbeat") long lastHeartbeat, + @Bind("expiresAt") long expiresAt); + + @SqlQuery("SELECT * FROM rdf_reindex_lock WHERE lockKey = :lockKey") + @RegisterRowMapper(RdfReindexLockMapper.class) + RdfReindexLockRecord findByKey(@Bind("lockKey") String lockKey); + + @SqlUpdate("DELETE FROM rdf_reindex_lock WHERE lockKey = :lockKey") + void delete(@Bind("lockKey") String lockKey); + + @SqlUpdate("DELETE FROM rdf_reindex_lock WHERE lockKey = :lockKey AND jobId = :jobId") + int deleteByKeyAndJob(@Bind("lockKey") String lockKey, @Bind("jobId") String jobId); + + @SqlUpdate("DELETE FROM rdf_reindex_lock WHERE expiresAt < :now") + int deleteExpiredLocks(@Bind("now") long now); + + @SqlUpdate( + "UPDATE rdf_reindex_lock SET jobId = :toJobId, serverId = :serverId, " + + "lastHeartbeat = :heartbeat, expiresAt = :expiresAt " + + "WHERE lockKey = :lockKey AND jobId = :fromJobId") + int updateLockOwner( + @Bind("lockKey") String lockKey, + @Bind("fromJobId") String fromJobId, + @Bind("toJobId") String toJobId, + @Bind("serverId") String serverId, + @Bind("heartbeat") long heartbeat, + @Bind("expiresAt") long expiresAt); + + default boolean tryAcquireLock( + String lockKey, String jobId, String serverId, long acquiredAt, long expiresAt) { + deleteExpiredLocks(System.currentTimeMillis()); + int inserted = insertIfNotExists(lockKey, jobId, serverId, acquiredAt, acquiredAt, expiresAt); + if (inserted > 0) { + return true; + } + + RdfReindexLockRecord existing = findByKey(lockKey); + if (existing != null && existing.isExpired()) { + delete(lockKey); + inserted = insertIfNotExists(lockKey, jobId, serverId, acquiredAt, acquiredAt, expiresAt); + return inserted > 0; + } + return false; + } + + default void releaseLock(String lockKey, String jobId) { + deleteByKeyAndJob(lockKey, jobId); + } + + default boolean transferLock( + String lockKey, + String fromJobId, + String toJobId, + String serverId, + long heartbeat, + long expiresAt) { + return updateLockOwner(lockKey, fromJobId, toJobId, serverId, heartbeat, expiresAt) > 0; + } + + class RdfReindexLockMapper implements RowMapper { + @Override + public RdfReindexLockRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new RdfReindexLockRecord( + rs.getString("lockKey"), + rs.getString("jobId"), + rs.getString("serverId"), + rs.getLong("acquiredAt"), + rs.getLong("lastHeartbeat"), + rs.getLong("expiresAt")); + } + } + + record RdfReindexLockRecord( + String lockKey, + String jobId, + String serverId, + long acquiredAt, + long lastHeartbeat, + long expiresAt) { + + public boolean isExpired() { + return System.currentTimeMillis() > expiresAt; + } + } + } + + /** DAO for RDF per-server distributed stats. */ + interface RdfIndexServerStatsDAO { + + record ServerStatsRecord( + String id, + String jobId, + String serverId, + String entityType, + long processedRecords, + long successRecords, + long failedRecords, + int partitionsCompleted, + int partitionsFailed, + long lastUpdatedAt) {} + + record AggregatedServerStats( + long processedRecords, + long successRecords, + long failedRecords, + int partitionsCompleted, + int partitionsFailed) {} + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO rdf_index_server_stats (id, jobId, serverId, entityType, processedRecords, " + + "successRecords, failedRecords, partitionsCompleted, partitionsFailed, lastUpdatedAt) " + + "VALUES (:id, :jobId, :serverId, :entityType, :processedRecords, :successRecords, " + + ":failedRecords, :partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " + + "ON DUPLICATE KEY UPDATE " + + "processedRecords = processedRecords + VALUES(processedRecords), " + + "successRecords = successRecords + VALUES(successRecords), " + + "failedRecords = failedRecords + VALUES(failedRecords), " + + "partitionsCompleted = partitionsCompleted + VALUES(partitionsCompleted), " + + "partitionsFailed = partitionsFailed + VALUES(partitionsFailed), " + + "lastUpdatedAt = VALUES(lastUpdatedAt)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO rdf_index_server_stats (id, jobId, serverId, entityType, processedRecords, " + + "successRecords, failedRecords, partitionsCompleted, partitionsFailed, lastUpdatedAt) " + + "VALUES (:id, :jobId, :serverId, :entityType, :processedRecords, :successRecords, " + + ":failedRecords, :partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " + + "ON CONFLICT (jobId, serverId, entityType) DO UPDATE SET " + + "processedRecords = rdf_index_server_stats.processedRecords + EXCLUDED.processedRecords, " + + "successRecords = rdf_index_server_stats.successRecords + EXCLUDED.successRecords, " + + "failedRecords = rdf_index_server_stats.failedRecords + EXCLUDED.failedRecords, " + + "partitionsCompleted = rdf_index_server_stats.partitionsCompleted + EXCLUDED.partitionsCompleted, " + + "partitionsFailed = rdf_index_server_stats.partitionsFailed + EXCLUDED.partitionsFailed, " + + "lastUpdatedAt = EXCLUDED.lastUpdatedAt", + connectionType = POSTGRES) + void incrementStats( + @Bind("id") String id, + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("entityType") String entityType, + @Bind("processedRecords") long processedRecords, + @Bind("successRecords") long successRecords, + @Bind("failedRecords") long failedRecords, + @Bind("partitionsCompleted") int partitionsCompleted, + @Bind("partitionsFailed") int partitionsFailed, + @Bind("lastUpdatedAt") long lastUpdatedAt); + + @SqlQuery("SELECT * FROM rdf_index_server_stats WHERE jobId = :jobId") + @RegisterRowMapper(RdfServerStatsRecordMapper.class) + List findByJobId(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT " + + "COALESCE(SUM(processedRecords), 0) as processedRecords, " + + "COALESCE(SUM(successRecords), 0) as successRecords, " + + "COALESCE(SUM(failedRecords), 0) as failedRecords, " + + "COALESCE(SUM(partitionsCompleted), 0) as partitionsCompleted, " + + "COALESCE(SUM(partitionsFailed), 0) as partitionsFailed " + + "FROM rdf_index_server_stats WHERE jobId = :jobId") + @RegisterRowMapper(RdfAggregatedServerStatsMapper.class) + AggregatedServerStats getAggregatedStats(@Bind("jobId") String jobId); + + @SqlUpdate("DELETE FROM rdf_index_server_stats") + void deleteAll(); + + class RdfServerStatsRecordMapper implements RowMapper { + @Override + public ServerStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new ServerStatsRecord( + rs.getString("id"), + rs.getString("jobId"), + rs.getString("serverId"), + rs.getString("entityType"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getInt("partitionsCompleted"), + rs.getInt("partitionsFailed"), + rs.getLong("lastUpdatedAt")); + } + } + + class RdfAggregatedServerStatsMapper implements RowMapper { + @Override + public AggregatedServerStats map(ResultSet rs, StatementContext ctx) throws SQLException { + return new AggregatedServerStats( + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getInt("partitionsCompleted"), + rs.getInt("partitionsFailed")); + } + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchReindexDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchReindexDAOs.java new file mode 100644 index 000000000000..5e52601da12e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchReindexDAOs.java @@ -0,0 +1,1599 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import lombok.Builder; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindBean; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.statement.SqlBatch; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; + +public interface SearchReindexDAOs { + @CreateSqlObject + SearchIndexJobDAO searchIndexJobDAO(); + + @CreateSqlObject + SearchIndexPartitionDAO searchIndexPartitionDAO(); + + @CreateSqlObject + SearchReindexLockDAO searchReindexLockDAO(); + + @CreateSqlObject + SearchIndexFailureDAO searchIndexFailureDAO(); + + @CreateSqlObject + SearchIndexRetryQueueDAO searchIndexRetryQueueDAO(); + + @CreateSqlObject + SearchIndexServerStatsDAO searchIndexServerStatsDAO(); + + /** DAO for distributed search index jobs */ + interface SearchIndexJobDAO { + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_job (id, status, jobConfiguration, targetIndexPrefix, totalRecords, " + + "processedRecords, successRecords, failedRecords, stats, createdBy, createdAt, updatedAt, " + + "registrationDeadline) " + + "VALUES (:id, :status, :jobConfiguration, :targetIndexPrefix, :totalRecords, " + + ":processedRecords, :successRecords, :failedRecords, :stats, :createdBy, :createdAt, :updatedAt, " + + ":registrationDeadline)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_job (id, status, jobConfiguration, targetIndexPrefix, totalRecords, " + + "processedRecords, successRecords, failedRecords, stats, createdBy, createdAt, updatedAt, " + + "registrationDeadline) " + + "VALUES (:id, :status, :jobConfiguration::jsonb, :targetIndexPrefix, :totalRecords, " + + ":processedRecords, :successRecords, :failedRecords, :stats::jsonb, :createdBy, :createdAt, :updatedAt, " + + ":registrationDeadline)", + connectionType = POSTGRES) + void insert( + @Bind("id") String id, + @Bind("status") String status, + @Bind("jobConfiguration") String jobConfiguration, + @Bind("targetIndexPrefix") String targetIndexPrefix, + @Bind("totalRecords") long totalRecords, + @Bind("processedRecords") long processedRecords, + @Bind("successRecords") long successRecords, + @Bind("failedRecords") long failedRecords, + @Bind("stats") String stats, + @Bind("createdBy") String createdBy, + @Bind("createdAt") long createdAt, + @Bind("updatedAt") long updatedAt, + @Bind("registrationDeadline") Long registrationDeadline); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE search_index_job SET status = :status, processedRecords = :processedRecords, " + + "successRecords = :successRecords, failedRecords = :failedRecords, stats = :stats, " + + "startedAt = :startedAt, completedAt = :completedAt, updatedAt = :updatedAt, " + + "errorMessage = :errorMessage WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE search_index_job SET status = :status, processedRecords = :processedRecords, " + + "successRecords = :successRecords, failedRecords = :failedRecords, stats = :stats::jsonb, " + + "startedAt = :startedAt, completedAt = :completedAt, updatedAt = :updatedAt, " + + "errorMessage = :errorMessage WHERE id = :id", + connectionType = POSTGRES) + void update( + @Bind("id") String id, + @Bind("status") String status, + @Bind("processedRecords") long processedRecords, + @Bind("successRecords") long successRecords, + @Bind("failedRecords") long failedRecords, + @Bind("stats") String stats, + @Bind("startedAt") Long startedAt, + @Bind("completedAt") Long completedAt, + @Bind("updatedAt") long updatedAt, + @Bind("errorMessage") String errorMessage); + + @SqlQuery("SELECT * FROM search_index_job WHERE id = :id") + @RegisterRowMapper(SearchIndexJobMapper.class) + SearchIndexJobRecord findById(@Bind("id") String id); + + @SqlQuery("SELECT * FROM search_index_job WHERE status IN () ORDER BY createdAt DESC") + @RegisterRowMapper(SearchIndexJobMapper.class) + List findByStatuses(@BindList("statuses") List statuses); + + @SqlQuery( + "SELECT * FROM search_index_job WHERE status IN () ORDER BY createdAt DESC LIMIT :limit") + @RegisterRowMapper(SearchIndexJobMapper.class) + List findByStatusesWithLimit( + @BindList("statuses") List statuses, @Bind("limit") int limit); + + @SqlQuery("SELECT * FROM search_index_job ORDER BY createdAt DESC LIMIT :limit") + @RegisterRowMapper(SearchIndexJobMapper.class) + List listRecent(@Bind("limit") int limit); + + @SqlUpdate("DELETE FROM search_index_job WHERE id = :id") + void delete(@Bind("id") String id); + + @SqlUpdate( + "DELETE FROM search_index_job WHERE status IN ('COMPLETED', 'FAILED', 'STOPPED') AND completedAt < :before") + int deleteOldJobs(@Bind("before") long before); + + @SqlUpdate("DELETE FROM search_index_job") + void deleteAll(); + + @SqlUpdate( + "UPDATE search_index_job SET registeredServerCount = :serverCount, updatedAt = :updatedAt WHERE id = :id") + void updateRegisteredServerCount( + @Bind("id") String id, + @Bind("serverCount") int serverCount, + @Bind("updatedAt") long updatedAt); + + @SqlQuery("SELECT registrationDeadline FROM search_index_job WHERE id = :id") + Long getRegistrationDeadline(@Bind("id") String id); + + @SqlQuery("SELECT registeredServerCount FROM search_index_job WHERE id = :id") + Integer getRegisteredServerCount(@Bind("id") String id); + + /** Get IDs of currently running jobs - lightweight query for polling */ + @SqlQuery("SELECT id FROM search_index_job WHERE status = 'RUNNING'") + List getRunningJobIds(); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE search_index_job SET stagedIndexMapping = :stagedIndexMapping, updatedAt = :updatedAt WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE search_index_job SET stagedIndexMapping = :stagedIndexMapping::jsonb, updatedAt = :updatedAt WHERE id = :id", + connectionType = POSTGRES) + void updateStagedIndexMapping( + @Bind("id") String id, + @Bind("stagedIndexMapping") String stagedIndexMapping, + @Bind("updatedAt") long updatedAt); + + @SqlUpdate("UPDATE search_index_job SET updatedAt = :updatedAt WHERE id = :id") + void touchJob(@Bind("id") String id, @Bind("updatedAt") long updatedAt); + + /** Row mapper for SearchIndexJobRecord */ + class SearchIndexJobMapper implements RowMapper { + @Override + public SearchIndexJobRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new SearchIndexJobRecord( + rs.getString("id"), + rs.getString("status"), + rs.getString("jobConfiguration"), + rs.getString("targetIndexPrefix"), + rs.getString("stagedIndexMapping"), + rs.getLong("totalRecords"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getString("stats"), + rs.getString("createdBy"), + rs.getLong("createdAt"), + (Long) rs.getObject("startedAt"), + (Long) rs.getObject("completedAt"), + rs.getLong("updatedAt"), + rs.getString("errorMessage"), + (Long) rs.getObject("registrationDeadline"), + (Integer) rs.getObject("registeredServerCount")); + } + } + + /** Record for job data from DB */ + record SearchIndexJobRecord( + String id, + String status, + String jobConfiguration, + String targetIndexPrefix, + String stagedIndexMapping, + long totalRecords, + long processedRecords, + long successRecords, + long failedRecords, + String stats, + String createdBy, + long createdAt, + Long startedAt, + Long completedAt, + long updatedAt, + String errorMessage, + Long registrationDeadline, + Integer registeredServerCount) {} + } + + /** DAO for distributed search index partitions */ + interface SearchIndexPartitionDAO { + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_partition (id, jobId, entityType, partitionIndex, rangeStart, rangeEnd, " + + "estimatedCount, workUnits, priority, status, processingCursor, claimableAt) " + + "VALUES (:id, :jobId, :entityType, :partitionIndex, :rangeStart, :rangeEnd, " + + ":estimatedCount, :workUnits, :priority, :status, :cursor, :claimableAt)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_partition (id, jobId, entityType, partitionIndex, rangeStart, rangeEnd, " + + "estimatedCount, workUnits, priority, status, processingCursor, claimableAt) " + + "VALUES (:id, :jobId, :entityType, :partitionIndex, :rangeStart, :rangeEnd, " + + ":estimatedCount, :workUnits, :priority, :status, :cursor, :claimableAt)", + connectionType = POSTGRES) + void insert( + @Bind("id") String id, + @Bind("jobId") String jobId, + @Bind("entityType") String entityType, + @Bind("partitionIndex") int partitionIndex, + @Bind("rangeStart") long rangeStart, + @Bind("rangeEnd") long rangeEnd, + @Bind("estimatedCount") long estimatedCount, + @Bind("workUnits") long workUnits, + @Bind("priority") int priority, + @Bind("status") String status, + @Bind("cursor") long cursor, + @Bind("claimableAt") long claimableAt); + + @SqlUpdate( + "UPDATE search_index_partition SET status = :status, processingCursor = :cursor, " + + "processedCount = :processedCount, successCount = :successCount, failedCount = :failedCount, " + + "assignedServer = :assignedServer, claimedAt = :claimedAt, startedAt = :startedAt, " + + "completedAt = :completedAt, lastUpdateAt = :lastUpdateAt, lastError = :lastError, " + + "retryCount = :retryCount WHERE id = :id") + void update( + @Bind("id") String id, + @Bind("status") String status, + @Bind("cursor") long cursor, + @Bind("processedCount") long processedCount, + @Bind("successCount") long successCount, + @Bind("failedCount") long failedCount, + @Bind("assignedServer") String assignedServer, + @Bind("claimedAt") Long claimedAt, + @Bind("startedAt") Long startedAt, + @Bind("completedAt") Long completedAt, + @Bind("lastUpdateAt") Long lastUpdateAt, + @Bind("lastError") String lastError, + @Bind("retryCount") int retryCount); + + @SqlUpdate( + "UPDATE search_index_partition SET processingCursor = :cursor, processedCount = :processedCount, " + + "successCount = :successCount, failedCount = :failedCount, lastUpdateAt = :lastUpdateAt " + + "WHERE id = :id") + void updateProgress( + @Bind("id") String id, + @Bind("cursor") long cursor, + @Bind("processedCount") long processedCount, + @Bind("successCount") long successCount, + @Bind("failedCount") long failedCount, + @Bind("lastUpdateAt") long lastUpdateAt); + + @SqlUpdate("UPDATE search_index_partition SET lastUpdateAt = :lastUpdateAt WHERE id = :id") + void updateHeartbeat(@Bind("id") String id, @Bind("lastUpdateAt") long lastUpdateAt); + + @SqlQuery("SELECT * FROM search_index_partition WHERE id = :id") + @RegisterRowMapper(SearchIndexPartitionMapper.class) + SearchIndexPartitionRecord findById(@Bind("id") String id); + + @SqlQuery( + "SELECT * FROM search_index_partition WHERE jobId = :jobId ORDER BY priority DESC, entityType, partitionIndex") + @RegisterRowMapper(SearchIndexPartitionMapper.class) + List findByJobId(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT * FROM search_index_partition WHERE jobId = :jobId AND status = 'PENDING' " + + "AND claimableAt <= :now " + + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED") + @RegisterRowMapper(SearchIndexPartitionMapper.class) + SearchIndexPartitionRecord findNextPendingPartitionForUpdate( + @Bind("jobId") String jobId, @Bind("now") long now); + + @SqlUpdate( + "UPDATE search_index_partition SET status = 'PROCESSING', " + + "assignedServer = :serverId, claimedAt = :now, startedAt = :now, lastUpdateAt = :now " + + "WHERE id = :partitionId AND status = 'PENDING'") + int claimPartitionById( + @Bind("partitionId") String partitionId, + @Bind("serverId") String serverId, + @Bind("now") long now); + + /** + * Atomically claim the next available partition using UPDATE with subquery. + * MySQL requires a JOIN-based approach since it doesn't allow subquery referencing same table. + * PostgreSQL can use direct subquery approach. + * Only claims partitions where claimableAt <= now (for staggered release). + */ + @ConnectionAwareSqlUpdate( + value = + "UPDATE search_index_partition p " + + "JOIN (SELECT id FROM search_index_partition WHERE jobId = :jobId AND status = 'PENDING' " + + "AND claimableAt <= :now " + + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED) t ON p.id = t.id " + + "SET p.status = 'PROCESSING', p.assignedServer = :serverId, p.claimedAt = :now, " + + "p.startedAt = :now, p.lastUpdateAt = :now", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE search_index_partition SET status = 'PROCESSING', " + + "assignedServer = :serverId, claimedAt = :now, startedAt = :now, lastUpdateAt = :now " + + "WHERE id = (SELECT id FROM search_index_partition WHERE jobId = :jobId AND status = 'PENDING' " + + "AND claimableAt <= :now " + + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1 FOR UPDATE SKIP LOCKED)", + connectionType = POSTGRES) + int claimNextPartitionAtomic( + @Bind("jobId") String jobId, @Bind("serverId") String serverId, @Bind("now") long now); + + @SqlQuery( + "SELECT * FROM search_index_partition WHERE jobId = :jobId AND status = 'PROCESSING' " + + "AND assignedServer = :serverId AND claimedAt = :claimedAt " + + "ORDER BY priority DESC, entityType, partitionIndex LIMIT 1") + @RegisterRowMapper(SearchIndexPartitionMapper.class) + SearchIndexPartitionRecord findLatestClaimedPartition( + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("claimedAt") long claimedAt); + + @SqlQuery( + "SELECT * FROM search_index_partition WHERE jobId = :jobId AND status = :status " + + "ORDER BY priority DESC, entityType, partitionIndex") + @RegisterRowMapper(SearchIndexPartitionMapper.class) + List findByJobIdAndStatus( + @Bind("jobId") String jobId, @Bind("status") String status); + + /** Count how many partitions a server currently has in PROCESSING status for a job */ + @SqlQuery( + "SELECT COUNT(*) FROM search_index_partition " + + "WHERE jobId = :jobId AND status = 'PROCESSING' AND assignedServer = :serverId") + int countInFlightPartitions(@Bind("jobId") String jobId, @Bind("serverId") String serverId); + + /** Count total PENDING partitions for a job */ + @SqlQuery( + "SELECT COUNT(*) FROM search_index_partition WHERE jobId = :jobId AND status = 'PENDING'") + int countPendingPartitions(@Bind("jobId") String jobId); + + /** Count total partitions for a job */ + @SqlQuery("SELECT COUNT(*) FROM search_index_partition WHERE jobId = :jobId") + int countTotalPartitions(@Bind("jobId") String jobId); + + /** Count partitions claimed by a specific server (PROCESSING or COMPLETED) */ + @SqlQuery( + "SELECT COUNT(*) FROM search_index_partition " + + "WHERE jobId = :jobId AND assignedServer = :serverId") + int countPartitionsClaimedByServer( + @Bind("jobId") String jobId, @Bind("serverId") String serverId); + + /** Count distinct servers that have claimed partitions for a job */ + @SqlQuery( + "SELECT COUNT(DISTINCT assignedServer) FROM search_index_partition " + + "WHERE jobId = :jobId AND assignedServer IS NOT NULL") + int countParticipatingServers(@Bind("jobId") String jobId); + + /** + * Reclaim stale partitions that can still be retried (under max retry limit). + * Returns the count of partitions reset to PENDING. + */ + @SqlUpdate( + "UPDATE search_index_partition SET status = 'PENDING', assignedServer = NULL, claimedAt = NULL, " + + "retryCount = retryCount + 1, lastError = 'Reclaimed due to stale heartbeat' " + + "WHERE jobId = :jobId AND status = 'PROCESSING' AND lastUpdateAt < :staleThreshold " + + "AND retryCount < :maxRetries") + int reclaimStalePartitionsForRetry( + @Bind("jobId") String jobId, + @Bind("staleThreshold") long staleThreshold, + @Bind("maxRetries") int maxRetries); + + /** + * Mark stale partitions that have exceeded retry limit as FAILED. + * Returns the count of partitions marked as failed. + */ + @SqlUpdate( + "UPDATE search_index_partition SET status = 'FAILED', " + + "lastError = 'Exceeded max retries after stale heartbeat', completedAt = :now " + + "WHERE jobId = :jobId AND status = 'PROCESSING' AND lastUpdateAt < :staleThreshold " + + "AND retryCount >= :maxRetries") + int failStalePartitionsExceedingRetries( + @Bind("jobId") String jobId, + @Bind("staleThreshold") long staleThreshold, + @Bind("maxRetries") int maxRetries, + @Bind("now") long now); + + @SqlUpdate( + "UPDATE search_index_partition SET status = 'CANCELLED' WHERE jobId = :jobId AND status = 'PENDING'") + int cancelPendingPartitions(@Bind("jobId") String jobId); + + @SqlUpdate( + "UPDATE search_index_partition SET status = 'CANCELLED', " + + "lastError = 'Stopped by user', completedAt = :now, lastUpdateAt = :now " + + "WHERE jobId = :jobId AND status IN ('PENDING','PROCESSING')") + int cancelInFlightPartitions(@Bind("jobId") String jobId, @Bind("now") long now); + + /** + * Status-guarded update: only mutates the row when it is still PROCESSING. Used by + * completion / failure paths so a late-arriving worker write cannot revert a CANCELLED + * row (set by requestStop) back to COMPLETED/FAILED. Returns the number of rows + * updated — 0 means another writer (typically requestStop) already moved the row to + * a terminal state and the caller should treat its update as a no-op. + */ + @SqlUpdate( + "UPDATE search_index_partition SET status = :status, processingCursor = :cursor, " + + "processedCount = :processedCount, successCount = :successCount, failedCount = :failedCount, " + + "assignedServer = :assignedServer, claimedAt = :claimedAt, startedAt = :startedAt, " + + "completedAt = :completedAt, lastUpdateAt = :lastUpdateAt, lastError = :lastError, " + + "retryCount = :retryCount WHERE id = :id AND status = 'PROCESSING'") + int updateIfProcessing( + @Bind("id") String id, + @Bind("status") String status, + @Bind("cursor") long cursor, + @Bind("processedCount") long processedCount, + @Bind("successCount") long successCount, + @Bind("failedCount") long failedCount, + @Bind("assignedServer") String assignedServer, + @Bind("claimedAt") Long claimedAt, + @Bind("startedAt") Long startedAt, + @Bind("completedAt") Long completedAt, + @Bind("lastUpdateAt") Long lastUpdateAt, + @Bind("lastError") String lastError, + @Bind("retryCount") int retryCount); + + @SqlQuery( + "SELECT * FROM search_index_partition WHERE jobId = :jobId AND status = 'PROCESSING' " + + "AND lastUpdateAt < :staleThreshold") + @RegisterRowMapper(SearchIndexPartitionMapper.class) + List findStalePartitions( + @Bind("jobId") String jobId, @Bind("staleThreshold") long staleThreshold); + + @SqlQuery( + "SELECT entityType, " + + "SUM(estimatedCount) as totalRecords, " + + "SUM(processedCount) as processedRecords, " + + "SUM(successCount) as successRecords, " + + "SUM(failedCount) as failedRecords, " + + "COUNT(*) as totalPartitions, " + + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " + + "SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) as failedPartitions " + + "FROM search_index_partition WHERE jobId = :jobId GROUP BY entityType") + @RegisterRowMapper(EntityStatsMapper.class) + List getEntityStats(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT " + + "SUM(estimatedCount) as totalRecords, " + + "SUM(processedCount) as processedRecords, " + + "SUM(successCount) as successRecords, " + + "SUM(failedCount) as failedRecords, " + + "COUNT(*) as totalPartitions, " + + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " + + "SUM(CASE WHEN status = 'FAILED' THEN 1 ELSE 0 END) as failedPartitions, " + + "SUM(CASE WHEN status = 'PENDING' THEN 1 ELSE 0 END) as pendingPartitions, " + + "SUM(CASE WHEN status = 'PROCESSING' THEN 1 ELSE 0 END) as processingPartitions " + + "FROM search_index_partition WHERE jobId = :jobId") + @RegisterRowMapper(AggregatedStatsMapper.class) + AggregatedStatsRecord getAggregatedStats(@Bind("jobId") String jobId); + + @SqlUpdate("DELETE FROM search_index_partition WHERE jobId = :jobId") + void deleteByJobId(@Bind("jobId") String jobId); + + @SqlUpdate("DELETE FROM search_index_partition") + void deleteAll(); + + @SqlQuery( + "SELECT assignedServer, " + + "SUM(processedCount) as processedRecords, " + + "SUM(successCount) as successRecords, " + + "SUM(failedCount) as failedRecords, " + + "COUNT(*) as totalPartitions, " + + "SUM(CASE WHEN status = 'COMPLETED' THEN 1 ELSE 0 END) as completedPartitions, " + + "SUM(CASE WHEN status = 'PROCESSING' THEN 1 ELSE 0 END) as processingPartitions " + + "FROM search_index_partition WHERE jobId = :jobId AND assignedServer IS NOT NULL " + + "GROUP BY assignedServer") + @RegisterRowMapper(ServerStatsMapper.class) + List getServerStats(@Bind("jobId") String jobId); + + /** Row mapper for partition records */ + class SearchIndexPartitionMapper implements RowMapper { + @Override + public SearchIndexPartitionRecord map(ResultSet rs, StatementContext ctx) + throws SQLException { + return new SearchIndexPartitionRecord( + rs.getString("id"), + rs.getString("jobId"), + rs.getString("entityType"), + rs.getInt("partitionIndex"), + rs.getLong("rangeStart"), + rs.getLong("rangeEnd"), + rs.getLong("estimatedCount"), + rs.getLong("workUnits"), + rs.getInt("priority"), + rs.getString("status"), + rs.getLong("processingCursor"), + rs.getLong("processedCount"), + rs.getLong("successCount"), + rs.getLong("failedCount"), + rs.getString("assignedServer"), + (Long) rs.getObject("claimedAt"), + (Long) rs.getObject("startedAt"), + (Long) rs.getObject("completedAt"), + (Long) rs.getObject("lastUpdateAt"), + rs.getString("lastError"), + rs.getInt("retryCount"), + rs.getLong("claimableAt")); + } + } + + /** Row mapper for entity stats */ + class EntityStatsMapper implements RowMapper { + @Override + public EntityStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new EntityStatsRecord( + rs.getString("entityType"), + rs.getLong("totalRecords"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getInt("totalPartitions"), + rs.getInt("completedPartitions"), + rs.getInt("failedPartitions")); + } + } + + /** Row mapper for aggregated stats */ + class AggregatedStatsMapper implements RowMapper { + @Override + public AggregatedStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new AggregatedStatsRecord( + rs.getLong("totalRecords"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getInt("totalPartitions"), + rs.getInt("completedPartitions"), + rs.getInt("failedPartitions"), + rs.getInt("pendingPartitions"), + rs.getInt("processingPartitions")); + } + } + + /** Record for partition data from DB */ + record SearchIndexPartitionRecord( + String id, + String jobId, + String entityType, + int partitionIndex, + long rangeStart, + long rangeEnd, + long estimatedCount, + long workUnits, + int priority, + String status, + long cursor, + long processedCount, + long successCount, + long failedCount, + String assignedServer, + Long claimedAt, + Long startedAt, + Long completedAt, + Long lastUpdateAt, + String lastError, + int retryCount, + long claimableAt) {} + + /** Record for entity stats aggregation */ + record EntityStatsRecord( + String entityType, + long totalRecords, + long processedRecords, + long successRecords, + long failedRecords, + int totalPartitions, + int completedPartitions, + int failedPartitions) {} + + /** Record for overall job stats aggregation */ + record AggregatedStatsRecord( + long totalRecords, + long processedRecords, + long successRecords, + long failedRecords, + int totalPartitions, + int completedPartitions, + int failedPartitions, + int pendingPartitions, + int processingPartitions) {} + + /** Record for per-server stats aggregation */ + record ServerStatsRecord( + String serverId, + long processedRecords, + long successRecords, + long failedRecords, + int totalPartitions, + int completedPartitions, + int processingPartitions) {} + + /** Row mapper for server stats */ + class ServerStatsMapper implements RowMapper { + @Override + public ServerStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new ServerStatsRecord( + rs.getString("assignedServer"), + rs.getLong("processedRecords"), + rs.getLong("successRecords"), + rs.getLong("failedRecords"), + rs.getInt("totalPartitions"), + rs.getInt("completedPartitions"), + rs.getInt("processingPartitions")); + } + } + + /** + * Record for partition quota statistics used in fair distribution. + * Includes both partition-count and work-based metrics for fair load balancing. + * + *

Work-based distribution ensures servers with high-record partitions don't + * monopolize the workload, even if partition counts appear balanced. + */ + record PartitionQuotaStats( + int inFlightCount, + int totalPartitions, + int claimedByServer, + int participatingServers, + int pendingPartitions, + long totalWorkUnits, + long workClaimedByServer, + long pendingWorkUnits) {} + + /** Row mapper for partition quota stats */ + class PartitionQuotaStatsMapper implements RowMapper { + @Override + public PartitionQuotaStats map(ResultSet rs, StatementContext ctx) throws SQLException { + return new PartitionQuotaStats( + rs.getInt("inFlightCount"), + rs.getInt("totalPartitions"), + rs.getInt("claimedByServer"), + rs.getInt("participatingServers"), + rs.getInt("pendingPartitions"), + rs.getLong("totalWorkUnits"), + rs.getLong("workClaimedByServer"), + rs.getLong("pendingWorkUnits")); + } + } + + /** + * Get all quota-related statistics in a single query for fair partition distribution. + * Includes both partition-count and work-based metrics. + */ + @SqlQuery( + "SELECT " + + "SUM(CASE WHEN status = 'PROCESSING' AND assignedServer = :serverId THEN 1 ELSE 0 END) as inFlightCount, " + + "COUNT(*) as totalPartitions, " + + "SUM(CASE WHEN assignedServer = :serverId THEN 1 ELSE 0 END) as claimedByServer, " + + "COUNT(DISTINCT CASE WHEN assignedServer IS NOT NULL THEN assignedServer END) as participatingServers, " + + "SUM(CASE WHEN status = 'PENDING' THEN 1 ELSE 0 END) as pendingPartitions, " + + "COALESCE(SUM(workUnits), 0) as totalWorkUnits, " + + "COALESCE(SUM(CASE WHEN assignedServer = :serverId THEN workUnits ELSE 0 END), 0) as workClaimedByServer, " + + "COALESCE(SUM(CASE WHEN status = 'PENDING' THEN workUnits ELSE 0 END), 0) as pendingWorkUnits " + + "FROM search_index_partition WHERE jobId = :jobId") + @RegisterRowMapper(PartitionQuotaStatsMapper.class) + PartitionQuotaStats getQuotaStats( + @Bind("jobId") String jobId, @Bind("serverId") String serverId); + + /** Get distinct servers that have claimed partitions for a job */ + @SqlQuery( + "SELECT DISTINCT assignedServer FROM search_index_partition " + + "WHERE jobId = :jobId AND assignedServer IS NOT NULL") + List getAssignedServers(@Bind("jobId") String jobId); + } + + /** DAO for distributed reindex lock */ + interface SearchReindexLockDAO { + + @SqlUpdate( + "INSERT INTO search_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " + + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt)") + void insert( + @Bind("lockKey") String lockKey, + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("acquiredAt") long acquiredAt, + @Bind("lastHeartbeat") long lastHeartbeat, + @Bind("expiresAt") long expiresAt); + + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO search_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " + + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_reindex_lock (lockKey, jobId, serverId, acquiredAt, lastHeartbeat, expiresAt) " + + "VALUES (:lockKey, :jobId, :serverId, :acquiredAt, :lastHeartbeat, :expiresAt) " + + "ON CONFLICT (lockKey) DO NOTHING", + connectionType = POSTGRES) + int insertIfNotExists( + @Bind("lockKey") String lockKey, + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("acquiredAt") long acquiredAt, + @Bind("lastHeartbeat") long lastHeartbeat, + @Bind("expiresAt") long expiresAt); + + @SqlUpdate( + "UPDATE search_reindex_lock SET lastHeartbeat = :lastHeartbeat, expiresAt = :expiresAt " + + "WHERE lockKey = :lockKey AND jobId = :jobId") + int updateHeartbeat( + @Bind("lockKey") String lockKey, + @Bind("jobId") String jobId, + @Bind("lastHeartbeat") long lastHeartbeat, + @Bind("expiresAt") long expiresAt); + + @SqlQuery("SELECT * FROM search_reindex_lock WHERE lockKey = :lockKey") + @RegisterRowMapper(SearchReindexLockMapper.class) + SearchReindexLockRecord findByKey(@Bind("lockKey") String lockKey); + + @SqlUpdate("DELETE FROM search_reindex_lock WHERE lockKey = :lockKey") + void delete(@Bind("lockKey") String lockKey); + + @SqlUpdate("DELETE FROM search_reindex_lock WHERE lockKey = :lockKey AND jobId = :jobId") + int deleteByKeyAndJob(@Bind("lockKey") String lockKey, @Bind("jobId") String jobId); + + @SqlUpdate("DELETE FROM search_reindex_lock WHERE expiresAt < :now") + int deleteExpiredLocks(@Bind("now") long now); + + /** + * Try to acquire a lock using atomic INSERT with conflict handling. Returns true if lock was + * acquired. + * + *

Uses database-level atomicity to prevent race conditions: + *

    + *
  • PostgreSQL: INSERT ... ON CONFLICT DO NOTHING + *
  • MySQL: INSERT IGNORE + *
+ * + *

If the insert fails due to a conflict, we check if the existing lock is expired and retry + * once after cleaning it up. + */ + default boolean tryAcquireLock( + String lockKey, String jobId, String serverId, long acquiredAt, long expiresAt) { + // First delete any expired locks + deleteExpiredLocks(System.currentTimeMillis()); + + // Atomically try to insert the lock - returns 1 if inserted, 0 if conflict + int inserted = insertIfNotExists(lockKey, jobId, serverId, acquiredAt, acquiredAt, expiresAt); + if (inserted > 0) { + return true; // Lock acquired successfully + } + + // Insert failed due to conflict - check if existing lock is expired + SearchReindexLockRecord existing = findByKey(lockKey); + if (existing != null && existing.isExpired()) { + // Lock is expired, delete it and retry once + delete(lockKey); + inserted = insertIfNotExists(lockKey, jobId, serverId, acquiredAt, acquiredAt, expiresAt); + return inserted > 0; + } + + // Lock is held by another active job + return false; + } + + /** Release a lock for a specific job */ + default void releaseLock(String lockKey, String jobId) { + deleteByKeyAndJob(lockKey, jobId); + } + + @SqlUpdate( + "UPDATE search_reindex_lock SET jobId = :toJobId, serverId = :serverId, " + + "lastHeartbeat = :heartbeat, expiresAt = :expiresAt " + + "WHERE lockKey = :lockKey AND jobId = :fromJobId") + int updateLockOwner( + @Bind("lockKey") String lockKey, + @Bind("fromJobId") String fromJobId, + @Bind("toJobId") String toJobId, + @Bind("serverId") String serverId, + @Bind("heartbeat") long heartbeat, + @Bind("expiresAt") long expiresAt); + + /** Atomically transfer a lock from one job to another */ + default boolean transferLock( + String lockKey, + String fromJobId, + String toJobId, + String serverId, + long heartbeat, + long expiresAt) { + int updated = updateLockOwner(lockKey, fromJobId, toJobId, serverId, heartbeat, expiresAt); + return updated > 0; + } + + /** Refresh a lock's heartbeat and expiration */ + default boolean refreshLock( + String lockKey, String jobId, String serverId, long heartbeat, long expiresAt) { + int updated = updateHeartbeat(lockKey, jobId, heartbeat, expiresAt); + return updated > 0; + } + + /** Clean up expired locks */ + default int cleanupExpiredLocks(long expirationThreshold) { + return deleteExpiredLocks(expirationThreshold); + } + + /** Get lock info for a specific lock key */ + default LockInfo getLockInfo(String lockKey) { + SearchReindexLockRecord record = findByKey(lockKey); + if (record == null) { + return null; + } + return new LockInfo( + record.lockKey(), + record.jobId(), + record.serverId(), + record.acquiredAt(), + record.lastHeartbeat(), + record.expiresAt()); + } + + /** Simple record for lock information */ + record LockInfo( + String lockKey, + String jobId, + String serverId, + long acquiredAt, + long lastHeartbeat, + long expiresAt) { + + public boolean isExpired() { + return System.currentTimeMillis() > expiresAt; + } + } + + /** Row mapper for lock records */ + class SearchReindexLockMapper implements RowMapper { + @Override + public SearchReindexLockRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new SearchReindexLockRecord( + rs.getString("lockKey"), + rs.getString("jobId"), + rs.getString("serverId"), + rs.getLong("acquiredAt"), + rs.getLong("lastHeartbeat"), + rs.getLong("expiresAt")); + } + } + + /** Record for lock data from DB */ + record SearchReindexLockRecord( + String lockKey, + String jobId, + String serverId, + long acquiredAt, + long lastHeartbeat, + long expiresAt) { + + public boolean isExpired() { + return System.currentTimeMillis() > expiresAt; + } + } + } + + /** DAO for search index failure records */ + interface SearchIndexFailureDAO { + + /** Bean class for @BindBean compatibility (records use id() not getId()) */ + @lombok.Getter + @lombok.AllArgsConstructor + class SearchIndexFailureRecord { + private final String id; + private final String jobId; + private final String serverId; + private final String entityType; + private final String entityId; + private final String entityFqn; + private final String failureStage; + private final String errorMessage; + private final String stackTrace; + private final long timestamp; + } + + @SqlUpdate( + "INSERT INTO search_index_failures (id, jobId, serverId, entityType, entityId, entityFqn, " + + "failureStage, errorMessage, stackTrace, timestamp) " + + "VALUES (:id, :jobId, :serverId, :entityType, :entityId, :entityFqn, " + + ":failureStage, :errorMessage, :stackTrace, :timestamp)") + void insert( + @Bind("id") String id, + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("entityType") String entityType, + @Bind("entityId") String entityId, + @Bind("entityFqn") String entityFqn, + @Bind("failureStage") String failureStage, + @Bind("errorMessage") String errorMessage, + @Bind("stackTrace") String stackTrace, + @Bind("timestamp") long timestamp); + + @SqlBatch( + "INSERT INTO search_index_failures (id, jobId, serverId, entityType, entityId, entityFqn, " + + "failureStage, errorMessage, stackTrace, timestamp) " + + "VALUES (:id, :jobId, :serverId, :entityType, :entityId, :entityFqn, " + + ":failureStage, :errorMessage, :stackTrace, :timestamp)") + void insertBatch(@BindBean List failures); + + @SqlQuery( + "SELECT * FROM search_index_failures WHERE serverId = :serverId " + + "ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + @RegisterRowMapper(SearchIndexFailureMapper.class) + List findByServerId( + @Bind("serverId") String serverId, @Bind("limit") int limit, @Bind("offset") int offset); + + @SqlQuery("SELECT COUNT(*) FROM search_index_failures WHERE serverId = :serverId") + int countByServerId(@Bind("serverId") String serverId); + + @SqlQuery( + "SELECT * FROM search_index_failures WHERE jobId = :jobId " + + "ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + @RegisterRowMapper(SearchIndexFailureMapper.class) + List findByJobId( + @Bind("jobId") String jobId, @Bind("limit") int limit, @Bind("offset") int offset); + + @SqlQuery("SELECT COUNT(*) FROM search_index_failures WHERE jobId = :jobId") + int countByJobId(@Bind("jobId") String jobId); + + /** + * Count only real failures for a job, excluding {@code READER_RELATIONSHIP_WARNING} rows — + * stale-relationship warnings are recorded for visibility but are not failures. + */ + @SqlQuery( + "SELECT COUNT(*) FROM search_index_failures WHERE jobId = :jobId " + + "AND failureStage <> 'READER_RELATIONSHIP_WARNING'") + int countFailuresByJobId(@Bind("jobId") String jobId); + + @SqlUpdate("DELETE FROM search_index_failures WHERE timestamp < :cutoffTime") + int deleteOlderThan(@Bind("cutoffTime") long cutoffTime); + + @SqlUpdate("DELETE FROM search_index_failures WHERE jobId = :jobId") + int deleteByJobId(@Bind("jobId") String jobId); + + @SqlUpdate("DELETE FROM search_index_failures") + int deleteAll(); + + @SqlQuery("SELECT COUNT(*) FROM search_index_failures") + int countAll(); + + @SqlQuery( + "SELECT * FROM search_index_failures ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + @RegisterRowMapper(SearchIndexFailureMapper.class) + List findAll(@Bind("limit") int limit, @Bind("offset") int offset); + + @SqlQuery("SELECT COUNT(*) FROM search_index_failures WHERE entityType = :entityType") + int countByEntityType(@Bind("entityType") String entityType); + + @SqlQuery( + "SELECT * FROM search_index_failures WHERE entityType = :entityType " + + "ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + @RegisterRowMapper(SearchIndexFailureMapper.class) + List findByEntityType( + @Bind("entityType") String entityType, + @Bind("limit") int limit, + @Bind("offset") int offset); + + class SearchIndexFailureMapper implements RowMapper { + @Override + public SearchIndexFailureRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new SearchIndexFailureRecord( + rs.getString("id"), + rs.getString("jobId"), + rs.getString("serverId"), + rs.getString("entityType"), + rs.getString("entityId"), + rs.getString("entityFqn"), + rs.getString("failureStage"), + rs.getString("errorMessage"), + rs.getString("stackTrace"), + rs.getLong("timestamp")); + } + } + } + + /** DAO for incremental search retry queue records. */ + interface SearchIndexRetryQueueDAO { + + @lombok.Getter + @lombok.AllArgsConstructor + class SearchIndexRetryRecord { + private final String entityId; + private final String entityFqn; + private final String failureReason; + private final String status; + private final String entityType; + private final int retryCount; + private final java.sql.Timestamp claimedAt; + } + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_retry_queue (entityId, entityFqn, failureReason, status, entityType) " + + "VALUES (:entityId, :entityFqn, :failureReason, :status, :entityType) " + + "ON DUPLICATE KEY UPDATE failureReason = VALUES(failureReason), status = VALUES(status), entityType = VALUES(entityType)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_retry_queue (entityId, entityFqn, failureReason, status, entityType) " + + "VALUES (:entityId, :entityFqn, :failureReason, :status, :entityType) " + + "ON CONFLICT (entityId, entityFqn) DO UPDATE SET " + + "failureReason = EXCLUDED.failureReason, status = EXCLUDED.status, entityType = EXCLUDED.entityType", + connectionType = POSTGRES) + void upsert( + @Bind("entityId") String entityId, + @Bind("entityFqn") String entityFqn, + @Bind("failureReason") String failureReason, + @Bind("status") String status, + @Bind("entityType") String entityType); + + @SqlQuery( + "SELECT entityId, entityFqn, failureReason, status, entityType, retryCount, claimedAt " + + "FROM search_index_retry_queue WHERE status = :status LIMIT :limit") + @RegisterRowMapper(SearchIndexRetryRecordMapper.class) + List findByStatus( + @Bind("status") String status, @Bind("limit") int limit); + + @SqlQuery( + "SELECT entityId, entityFqn, failureReason, status, entityType, retryCount, claimedAt " + + "FROM search_index_retry_queue WHERE status IN () LIMIT :limit") + @RegisterRowMapper(SearchIndexRetryRecordMapper.class) + List findByStatuses( + @BindList("statuses") List statuses, @Bind("limit") int limit); + + @SqlUpdate( + "UPDATE search_index_retry_queue SET status = :newStatus " + + "WHERE entityId = :entityId AND entityFqn = :entityFqn AND status = :currentStatus") + int updateStatusIfCurrent( + @Bind("entityId") String entityId, + @Bind("entityFqn") String entityFqn, + @Bind("currentStatus") String currentStatus, + @Bind("newStatus") String newStatus); + + @SqlUpdate( + "UPDATE search_index_retry_queue SET status = :status, failureReason = :failureReason " + + "WHERE entityId = :entityId AND entityFqn = :entityFqn") + int updateFailureAndStatus( + @Bind("entityId") String entityId, + @Bind("entityFqn") String entityFqn, + @Bind("failureReason") String failureReason, + @Bind("status") String status); + + @SqlUpdate( + "UPDATE search_index_retry_queue SET status = :status " + + "WHERE entityId = :entityId AND entityFqn = :entityFqn") + int updateStatus( + @Bind("entityId") String entityId, + @Bind("entityFqn") String entityFqn, + @Bind("status") String status); + + @SqlUpdate( + "DELETE FROM search_index_retry_queue WHERE entityId = :entityId AND entityFqn = :entityFqn") + int deleteByEntity(@Bind("entityId") String entityId, @Bind("entityFqn") String entityFqn); + + @SqlUpdate("DELETE FROM search_index_retry_queue WHERE status IN ()") + int deleteByStatuses(@BindList("statuses") List statuses); + + @SqlQuery("SELECT COUNT(*) FROM search_index_retry_queue WHERE status = :status") + int countByStatus(@Bind("status") String status); + + @SqlUpdate( + "UPDATE search_index_retry_queue SET status = 'IN_PROGRESS', claimedAt = NOW() " + + "WHERE entityId = :entityId AND entityFqn = :entityFqn AND status = :currentStatus") + int claimRecord( + @Bind("entityId") String entityId, + @Bind("entityFqn") String entityFqn, + @Bind("currentStatus") String currentStatus); + + @SqlUpdate( + "UPDATE search_index_retry_queue SET status = 'PENDING', claimedAt = NULL " + + "WHERE status = 'IN_PROGRESS' AND claimedAt < :cutoff") + int recoverStaleInProgress(@Bind("cutoff") java.sql.Timestamp cutoff); + + @SqlUpdate( + "UPDATE search_index_retry_queue SET status = :status, failureReason = :failureReason, " + + "retryCount = retryCount + 1, claimedAt = NULL " + + "WHERE entityId = :entityId AND entityFqn = :entityFqn") + int updateFailureAndRetryCount( + @Bind("entityId") String entityId, + @Bind("entityFqn") String entityFqn, + @Bind("failureReason") String failureReason, + @Bind("status") String status); + + default List claimPending(int batchSize) { + int fetchSize = Math.max(batchSize * 5, batchSize); + List candidates = + new ArrayList<>( + findByStatuses(List.of("PENDING", "PENDING_RETRY_1", "PENDING_RETRY_2"), fetchSize)); + // Shuffle so concurrent worker threads attempt different rows first, + // reducing wasted optimistic-lock failures on the same candidates. + Collections.shuffle(candidates); + List claimed = new ArrayList<>(); + for (SearchIndexRetryRecord candidate : candidates) { + if (claimed.size() >= batchSize) { + break; + } + int updated = + claimRecord(candidate.getEntityId(), candidate.getEntityFqn(), candidate.getStatus()); + if (updated == 1) { + claimed.add(candidate); + } + } + return claimed; + } + + @SqlQuery( + "SELECT entityId, entityFqn, failureReason, status, entityType, retryCount, claimedAt " + + "FROM search_index_retry_queue ORDER BY retryCount DESC, claimedAt DESC " + + "LIMIT :limit OFFSET :offset") + @RegisterRowMapper(SearchIndexRetryRecordMapper.class) + List listAll(@Bind("limit") int limit, @Bind("offset") int offset); + + @SqlQuery("SELECT COUNT(*) FROM search_index_retry_queue") + int countAll(); + + class SearchIndexRetryRecordMapper implements RowMapper { + @Override + public SearchIndexRetryRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new SearchIndexRetryRecord( + rs.getString("entityId"), + rs.getString("entityFqn"), + rs.getString("failureReason"), + rs.getString("status"), + rs.getString("entityType"), + rs.getInt("retryCount"), + rs.getTimestamp("claimedAt")); + } + } + } + + /** DAO for search index per-server stats in distributed mode */ + interface SearchIndexServerStatsDAO { + + record ServerStatsRecord( + String id, + String jobId, + String serverId, + String entityType, + long readerSuccess, + long readerFailed, + long readerWarnings, + long sinkSuccess, + long sinkFailed, + long processSuccess, + long processFailed, + long vectorSuccess, + long vectorFailed, + long readerTimeMs, + long processTimeMs, + long sinkTimeMs, + long vectorTimeMs, + int partitionsCompleted, + int partitionsFailed, + long lastUpdatedAt) {} + + record AggregatedServerStats( + long readerSuccess, + long readerFailed, + long readerWarnings, + long sinkSuccess, + long sinkFailed, + long processSuccess, + long processFailed, + long vectorSuccess, + long vectorFailed, + long readerTimeMs, + long processTimeMs, + long sinkTimeMs, + long vectorTimeMs, + int partitionsCompleted, + int partitionsFailed) {} + + record EntityStats( + String entityType, + long readerSuccess, + long readerFailed, + long readerWarnings, + long sinkSuccess, + long sinkFailed, + long processSuccess, + long processFailed, + long vectorSuccess, + long vectorFailed, + long readerTimeMs, + long processTimeMs, + long sinkTimeMs, + long vectorTimeMs) {} + + /** + * Increment stats using delta values. This is the primary method for updating stats - + * it adds the delta values to existing values, creating the row if it doesn't exist. + */ + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_server_stats (id, jobId, serverId, entityType, " + + "readerSuccess, readerFailed, readerWarnings, sinkSuccess, sinkFailed, " + + "processSuccess, processFailed, vectorSuccess, vectorFailed, " + + "readerTimeMs, processTimeMs, sinkTimeMs, vectorTimeMs, " + + "partitionsCompleted, partitionsFailed, lastUpdatedAt) " + + "VALUES (:id, :jobId, :serverId, :entityType, " + + ":readerSuccess, :readerFailed, :readerWarnings, :sinkSuccess, :sinkFailed, " + + ":processSuccess, :processFailed, :vectorSuccess, :vectorFailed, " + + ":readerTimeMs, :processTimeMs, :sinkTimeMs, :vectorTimeMs, " + + ":partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " + + "ON DUPLICATE KEY UPDATE " + + "readerSuccess = readerSuccess + VALUES(readerSuccess), " + + "readerFailed = readerFailed + VALUES(readerFailed), " + + "readerWarnings = readerWarnings + VALUES(readerWarnings), " + + "sinkSuccess = sinkSuccess + VALUES(sinkSuccess), " + + "sinkFailed = sinkFailed + VALUES(sinkFailed), " + + "processSuccess = processSuccess + VALUES(processSuccess), " + + "processFailed = processFailed + VALUES(processFailed), " + + "vectorSuccess = vectorSuccess + VALUES(vectorSuccess), " + + "vectorFailed = vectorFailed + VALUES(vectorFailed), " + + "readerTimeMs = readerTimeMs + VALUES(readerTimeMs), " + + "processTimeMs = processTimeMs + VALUES(processTimeMs), " + + "sinkTimeMs = sinkTimeMs + VALUES(sinkTimeMs), " + + "vectorTimeMs = vectorTimeMs + VALUES(vectorTimeMs), " + + "partitionsCompleted = partitionsCompleted + VALUES(partitionsCompleted), " + + "partitionsFailed = partitionsFailed + VALUES(partitionsFailed), " + + "lastUpdatedAt = VALUES(lastUpdatedAt)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_server_stats (id, jobId, serverId, entityType, " + + "readerSuccess, readerFailed, readerWarnings, sinkSuccess, sinkFailed, " + + "processSuccess, processFailed, vectorSuccess, vectorFailed, " + + "readerTimeMs, processTimeMs, sinkTimeMs, vectorTimeMs, " + + "partitionsCompleted, partitionsFailed, lastUpdatedAt) " + + "VALUES (:id, :jobId, :serverId, :entityType, " + + ":readerSuccess, :readerFailed, :readerWarnings, :sinkSuccess, :sinkFailed, " + + ":processSuccess, :processFailed, :vectorSuccess, :vectorFailed, " + + ":readerTimeMs, :processTimeMs, :sinkTimeMs, :vectorTimeMs, " + + ":partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " + + "ON CONFLICT (jobId, serverId, entityType) DO UPDATE SET " + + "readerSuccess = search_index_server_stats.readerSuccess + EXCLUDED.readerSuccess, " + + "readerFailed = search_index_server_stats.readerFailed + EXCLUDED.readerFailed, " + + "readerWarnings = search_index_server_stats.readerWarnings + EXCLUDED.readerWarnings, " + + "sinkSuccess = search_index_server_stats.sinkSuccess + EXCLUDED.sinkSuccess, " + + "sinkFailed = search_index_server_stats.sinkFailed + EXCLUDED.sinkFailed, " + + "processSuccess = search_index_server_stats.processSuccess + EXCLUDED.processSuccess, " + + "processFailed = search_index_server_stats.processFailed + EXCLUDED.processFailed, " + + "vectorSuccess = search_index_server_stats.vectorSuccess + EXCLUDED.vectorSuccess, " + + "vectorFailed = search_index_server_stats.vectorFailed + EXCLUDED.vectorFailed, " + + "readerTimeMs = search_index_server_stats.readerTimeMs + EXCLUDED.readerTimeMs, " + + "processTimeMs = search_index_server_stats.processTimeMs + EXCLUDED.processTimeMs, " + + "sinkTimeMs = search_index_server_stats.sinkTimeMs + EXCLUDED.sinkTimeMs, " + + "vectorTimeMs = search_index_server_stats.vectorTimeMs + EXCLUDED.vectorTimeMs, " + + "partitionsCompleted = search_index_server_stats.partitionsCompleted + EXCLUDED.partitionsCompleted, " + + "partitionsFailed = search_index_server_stats.partitionsFailed + EXCLUDED.partitionsFailed, " + + "lastUpdatedAt = EXCLUDED.lastUpdatedAt", + connectionType = POSTGRES) + void incrementStats( + @Bind("id") String id, + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("entityType") String entityType, + @Bind("readerSuccess") long readerSuccess, + @Bind("readerFailed") long readerFailed, + @Bind("readerWarnings") long readerWarnings, + @Bind("sinkSuccess") long sinkSuccess, + @Bind("sinkFailed") long sinkFailed, + @Bind("processSuccess") long processSuccess, + @Bind("processFailed") long processFailed, + @Bind("vectorSuccess") long vectorSuccess, + @Bind("vectorFailed") long vectorFailed, + @Bind("readerTimeMs") long readerTimeMs, + @Bind("processTimeMs") long processTimeMs, + @Bind("sinkTimeMs") long sinkTimeMs, + @Bind("vectorTimeMs") long vectorTimeMs, + @Bind("partitionsCompleted") int partitionsCompleted, + @Bind("partitionsFailed") int partitionsFailed, + @Bind("lastUpdatedAt") long lastUpdatedAt); + + /** + * Replace stats with absolute values. Used by distributed coordinator to persist + * aggregate stats for the server. + */ + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_server_stats (id, jobId, serverId, entityType, " + + "readerSuccess, readerFailed, readerWarnings, sinkSuccess, sinkFailed, " + + "processSuccess, processFailed, vectorSuccess, vectorFailed, " + + "readerTimeMs, processTimeMs, sinkTimeMs, vectorTimeMs, " + + "partitionsCompleted, partitionsFailed, lastUpdatedAt) " + + "VALUES (:id, :jobId, :serverId, :entityType, " + + ":readerSuccess, :readerFailed, :readerWarnings, :sinkSuccess, :sinkFailed, " + + ":processSuccess, :processFailed, :vectorSuccess, :vectorFailed, " + + ":readerTimeMs, :processTimeMs, :sinkTimeMs, :vectorTimeMs, " + + ":partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " + + "ON DUPLICATE KEY UPDATE " + + "readerSuccess = VALUES(readerSuccess), " + + "readerFailed = VALUES(readerFailed), " + + "readerWarnings = VALUES(readerWarnings), " + + "sinkSuccess = VALUES(sinkSuccess), " + + "sinkFailed = VALUES(sinkFailed), " + + "processSuccess = VALUES(processSuccess), " + + "processFailed = VALUES(processFailed), " + + "vectorSuccess = VALUES(vectorSuccess), " + + "vectorFailed = VALUES(vectorFailed), " + + "readerTimeMs = VALUES(readerTimeMs), " + + "processTimeMs = VALUES(processTimeMs), " + + "sinkTimeMs = VALUES(sinkTimeMs), " + + "vectorTimeMs = VALUES(vectorTimeMs), " + + "partitionsCompleted = VALUES(partitionsCompleted), " + + "partitionsFailed = VALUES(partitionsFailed), " + + "lastUpdatedAt = VALUES(lastUpdatedAt)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO search_index_server_stats (id, jobId, serverId, entityType, " + + "readerSuccess, readerFailed, readerWarnings, sinkSuccess, sinkFailed, " + + "processSuccess, processFailed, vectorSuccess, vectorFailed, " + + "readerTimeMs, processTimeMs, sinkTimeMs, vectorTimeMs, " + + "partitionsCompleted, partitionsFailed, lastUpdatedAt) " + + "VALUES (:id, :jobId, :serverId, :entityType, " + + ":readerSuccess, :readerFailed, :readerWarnings, :sinkSuccess, :sinkFailed, " + + ":processSuccess, :processFailed, :vectorSuccess, :vectorFailed, " + + ":readerTimeMs, :processTimeMs, :sinkTimeMs, :vectorTimeMs, " + + ":partitionsCompleted, :partitionsFailed, :lastUpdatedAt) " + + "ON CONFLICT (jobId, serverId, entityType) DO UPDATE SET " + + "readerSuccess = EXCLUDED.readerSuccess, " + + "readerFailed = EXCLUDED.readerFailed, " + + "readerWarnings = EXCLUDED.readerWarnings, " + + "sinkSuccess = EXCLUDED.sinkSuccess, " + + "sinkFailed = EXCLUDED.sinkFailed, " + + "processSuccess = EXCLUDED.processSuccess, " + + "processFailed = EXCLUDED.processFailed, " + + "vectorSuccess = EXCLUDED.vectorSuccess, " + + "vectorFailed = EXCLUDED.vectorFailed, " + + "readerTimeMs = EXCLUDED.readerTimeMs, " + + "processTimeMs = EXCLUDED.processTimeMs, " + + "sinkTimeMs = EXCLUDED.sinkTimeMs, " + + "vectorTimeMs = EXCLUDED.vectorTimeMs, " + + "partitionsCompleted = EXCLUDED.partitionsCompleted, " + + "partitionsFailed = EXCLUDED.partitionsFailed, " + + "lastUpdatedAt = EXCLUDED.lastUpdatedAt", + connectionType = POSTGRES) + void replaceStats( + @Bind("id") String id, + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("entityType") String entityType, + @Bind("readerSuccess") long readerSuccess, + @Bind("readerFailed") long readerFailed, + @Bind("readerWarnings") long readerWarnings, + @Bind("sinkSuccess") long sinkSuccess, + @Bind("sinkFailed") long sinkFailed, + @Bind("processSuccess") long processSuccess, + @Bind("processFailed") long processFailed, + @Bind("vectorSuccess") long vectorSuccess, + @Bind("vectorFailed") long vectorFailed, + @Bind("readerTimeMs") long readerTimeMs, + @Bind("processTimeMs") long processTimeMs, + @Bind("sinkTimeMs") long sinkTimeMs, + @Bind("vectorTimeMs") long vectorTimeMs, + @Bind("partitionsCompleted") int partitionsCompleted, + @Bind("partitionsFailed") int partitionsFailed, + @Bind("lastUpdatedAt") long lastUpdatedAt); + + @SqlQuery("SELECT * FROM search_index_server_stats WHERE jobId = :jobId") + @RegisterRowMapper(ServerStatsMapper.class) + List findByJobId(@Bind("jobId") String jobId); + + @SqlQuery( + "SELECT * FROM search_index_server_stats WHERE jobId = :jobId AND serverId = :serverId AND entityType = :entityType") + @RegisterRowMapper(ServerStatsMapper.class) + ServerStatsRecord findByJobIdServerIdEntityType( + @Bind("jobId") String jobId, + @Bind("serverId") String serverId, + @Bind("entityType") String entityType); + + /** Get aggregated stats across all servers and entity types for a job */ + @SqlQuery( + "SELECT " + + "COALESCE(SUM(readerSuccess), 0) as readerSuccess, " + + "COALESCE(SUM(readerFailed), 0) as readerFailed, " + + "COALESCE(SUM(readerWarnings), 0) as readerWarnings, " + + "COALESCE(SUM(sinkSuccess), 0) as sinkSuccess, " + + "COALESCE(SUM(sinkFailed), 0) as sinkFailed, " + + "COALESCE(SUM(processSuccess), 0) as processSuccess, " + + "COALESCE(SUM(processFailed), 0) as processFailed, " + + "COALESCE(SUM(vectorSuccess), 0) as vectorSuccess, " + + "COALESCE(SUM(vectorFailed), 0) as vectorFailed, " + + "COALESCE(SUM(readerTimeMs), 0) as readerTimeMs, " + + "COALESCE(SUM(processTimeMs), 0) as processTimeMs, " + + "COALESCE(SUM(sinkTimeMs), 0) as sinkTimeMs, " + + "COALESCE(SUM(vectorTimeMs), 0) as vectorTimeMs, " + + "COALESCE(SUM(partitionsCompleted), 0) as partitionsCompleted, " + + "COALESCE(SUM(partitionsFailed), 0) as partitionsFailed " + + "FROM search_index_server_stats WHERE jobId = :jobId") + @RegisterRowMapper(AggregatedServerStatsMapper.class) + AggregatedServerStats getAggregatedStats(@Bind("jobId") String jobId); + + /** Get stats grouped by entity type for a job */ + @SqlQuery( + "SELECT entityType, " + + "COALESCE(SUM(readerSuccess), 0) as readerSuccess, " + + "COALESCE(SUM(readerFailed), 0) as readerFailed, " + + "COALESCE(SUM(readerWarnings), 0) as readerWarnings, " + + "COALESCE(SUM(sinkSuccess), 0) as sinkSuccess, " + + "COALESCE(SUM(sinkFailed), 0) as sinkFailed, " + + "COALESCE(SUM(processSuccess), 0) as processSuccess, " + + "COALESCE(SUM(processFailed), 0) as processFailed, " + + "COALESCE(SUM(vectorSuccess), 0) as vectorSuccess, " + + "COALESCE(SUM(vectorFailed), 0) as vectorFailed, " + + "COALESCE(SUM(readerTimeMs), 0) as readerTimeMs, " + + "COALESCE(SUM(processTimeMs), 0) as processTimeMs, " + + "COALESCE(SUM(sinkTimeMs), 0) as sinkTimeMs, " + + "COALESCE(SUM(vectorTimeMs), 0) as vectorTimeMs " + + "FROM search_index_server_stats WHERE jobId = :jobId " + + "GROUP BY entityType") + @RegisterRowMapper(EntityStatsMapper.class) + List getStatsByEntityType(@Bind("jobId") String jobId); + + /** + * Per-server timing breakdown. Sums every counter and timing column for each serverId, + * letting the UI show "is one node dragging the cluster" for distributed runs. + */ + record ServerTimingStats( + String serverId, + long readerSuccess, + long sinkSuccess, + long processSuccess, + long vectorSuccess, + long readerTimeMs, + long processTimeMs, + long sinkTimeMs, + long vectorTimeMs) {} + + @SqlQuery( + "SELECT serverId, " + + "COALESCE(SUM(readerSuccess), 0) as readerSuccess, " + + "COALESCE(SUM(sinkSuccess), 0) as sinkSuccess, " + + "COALESCE(SUM(processSuccess), 0) as processSuccess, " + + "COALESCE(SUM(vectorSuccess), 0) as vectorSuccess, " + + "COALESCE(SUM(readerTimeMs), 0) as readerTimeMs, " + + "COALESCE(SUM(processTimeMs), 0) as processTimeMs, " + + "COALESCE(SUM(sinkTimeMs), 0) as sinkTimeMs, " + + "COALESCE(SUM(vectorTimeMs), 0) as vectorTimeMs " + + "FROM search_index_server_stats WHERE jobId = :jobId " + + "GROUP BY serverId") + @RegisterRowMapper(ServerTimingStatsMapper.class) + List getStatsByServer(@Bind("jobId") String jobId); + + class ServerTimingStatsMapper implements RowMapper { + @Override + public ServerTimingStats map(ResultSet rs, StatementContext ctx) throws SQLException { + return new ServerTimingStats( + rs.getString("serverId"), + rs.getLong("readerSuccess"), + rs.getLong("sinkSuccess"), + rs.getLong("processSuccess"), + rs.getLong("vectorSuccess"), + rs.getLong("readerTimeMs"), + rs.getLong("processTimeMs"), + rs.getLong("sinkTimeMs"), + rs.getLong("vectorTimeMs")); + } + } + + @SqlUpdate("DELETE FROM search_index_server_stats WHERE jobId = :jobId") + void deleteByJobId(@Bind("jobId") String jobId); + + @SqlUpdate("DELETE FROM search_index_server_stats") + void deleteAll(); + + class ServerStatsMapper implements RowMapper { + @Override + public ServerStatsRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new ServerStatsRecord( + rs.getString("id"), + rs.getString("jobId"), + rs.getString("serverId"), + rs.getString("entityType"), + rs.getLong("readerSuccess"), + rs.getLong("readerFailed"), + rs.getLong("readerWarnings"), + rs.getLong("sinkSuccess"), + rs.getLong("sinkFailed"), + rs.getLong("processSuccess"), + rs.getLong("processFailed"), + rs.getLong("vectorSuccess"), + rs.getLong("vectorFailed"), + rs.getLong("readerTimeMs"), + rs.getLong("processTimeMs"), + rs.getLong("sinkTimeMs"), + rs.getLong("vectorTimeMs"), + rs.getInt("partitionsCompleted"), + rs.getInt("partitionsFailed"), + rs.getLong("lastUpdatedAt")); + } + } + + class AggregatedServerStatsMapper implements RowMapper { + @Override + public AggregatedServerStats map(ResultSet rs, StatementContext ctx) throws SQLException { + return new AggregatedServerStats( + rs.getLong("readerSuccess"), + rs.getLong("readerFailed"), + rs.getLong("readerWarnings"), + rs.getLong("sinkSuccess"), + rs.getLong("sinkFailed"), + rs.getLong("processSuccess"), + rs.getLong("processFailed"), + rs.getLong("vectorSuccess"), + rs.getLong("vectorFailed"), + rs.getLong("readerTimeMs"), + rs.getLong("processTimeMs"), + rs.getLong("sinkTimeMs"), + rs.getLong("vectorTimeMs"), + rs.getInt("partitionsCompleted"), + rs.getInt("partitionsFailed")); + } + } + + class EntityStatsMapper implements RowMapper { + @Override + public EntityStats map(ResultSet rs, StatementContext ctx) throws SQLException { + return new EntityStats( + rs.getString("entityType"), + rs.getLong("readerSuccess"), + rs.getLong("readerFailed"), + rs.getLong("readerWarnings"), + rs.getLong("sinkSuccess"), + rs.getLong("sinkFailed"), + rs.getLong("processSuccess"), + rs.getLong("processFailed"), + rs.getLong("vectorSuccess"), + rs.getLong("vectorFailed"), + rs.getLong("readerTimeMs"), + rs.getLong("processTimeMs"), + rs.getLong("sinkTimeMs"), + rs.getLong("vectorTimeMs")); + } + } + } + + @Builder + record ActivityStreamRow( + String id, + String eventType, + String entityType, + String entityId, + String entityFqnHash, + String about, + String aboutFqnHash, + String actorId, + String actorName, + Long timestamp, + String summary, + String fieldName, + String oldValue, + String newValue, + String domains, + String json) {} +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java index bc23d342b306..e5d5faf4159b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ServiceEntityRepository.java @@ -187,7 +187,8 @@ public T addTestConnectionResult(UUID serviceId, TestConnectionResult testConnec dao.update(serviceId, service.getFullyQualifiedName(), JsonUtils.pojoToJson(service)); // Direct dao.update skips invalidateCachesAfterStore, so the next read would serve the // pre-test-connection JSON from cache. Drop every cached variant for this service. - invalidateCacheForEntity(entityType, serviceId, service.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + entityType, serviceId, service.getFullyQualifiedName()); return service; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index 5cfc44dd2ceb..d96f2b03535b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -72,7 +72,7 @@ import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.governance.workflows.WorkflowHandler; -import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO; +import org.openmetadata.service.jdbi3.SystemTokenDAOs.SystemDAO; import org.openmetadata.service.logstorage.LogStorageFactory; import org.openmetadata.service.logstorage.LogStorageInterface; import org.openmetadata.service.migration.MigrationValidationClient; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemTokenDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemTokenDAOs.java new file mode 100644 index 000000000000..71eebdeb2299 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemTokenDAOs.java @@ -0,0 +1,442 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.UUID; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.core.statement.StatementException; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.openmetadata.api.configuration.UiThemePreference; +import org.openmetadata.schema.TokenInterface; +import org.openmetadata.schema.api.configuration.LoginConfiguration; +import org.openmetadata.schema.api.configuration.MCPConfiguration; +import org.openmetadata.schema.api.configuration.OpenMetadataBaseUrlConfiguration; +import org.openmetadata.schema.api.configuration.profiler.ProfilerConfiguration; +import org.openmetadata.schema.api.lineage.LineageSettings; +import org.openmetadata.schema.api.search.SearchSettings; +import org.openmetadata.schema.api.security.AuthenticationConfiguration; +import org.openmetadata.schema.api.security.AuthorizerConfiguration; +import org.openmetadata.schema.auth.EmailVerificationToken; +import org.openmetadata.schema.auth.PasswordResetToken; +import org.openmetadata.schema.auth.PersonalAccessToken; +import org.openmetadata.schema.auth.RefreshToken; +import org.openmetadata.schema.auth.TokenType; +import org.openmetadata.schema.auth.collate.SupportToken; +import org.openmetadata.schema.configuration.AssetCertificationSettings; +import org.openmetadata.schema.configuration.EntityRulesSettings; +import org.openmetadata.schema.configuration.GlossaryTermRelationSettings; +import org.openmetadata.schema.configuration.OpenLineageSettings; +import org.openmetadata.schema.configuration.WorkflowSettings; +import org.openmetadata.schema.email.SmtpSettings; +import org.openmetadata.schema.security.scim.ScimConfiguration; +import org.openmetadata.schema.service.configuration.teamsApp.TeamsAppConfiguration; +import org.openmetadata.schema.settings.Settings; +import org.openmetadata.schema.settings.SettingsType; +import org.openmetadata.schema.util.EntitiesCount; +import org.openmetadata.schema.util.ServicesCount; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.jdbi3.oauth.OAuthRecords; +import org.openmetadata.service.security.session.UserSession; +import org.openmetadata.service.util.jdbi.BindUUID; + +public interface SystemTokenDAOs { + @CreateSqlObject + SystemDAO systemDAO(); + + @CreateSqlObject + TokenDAO getTokenDAO(); + + @CreateSqlObject + UserSessionDAO getUserSessionDAO(); + + interface SystemDAO { + @ConnectionAwareSqlQuery( + value = + "SELECT (SELECT COUNT(fqnHash) FROM table_entity ) as tableCount, " + + "(SELECT COUNT(fqnHash) FROM topic_entity ) as topicCount, " + + "(SELECT COUNT(fqnHash) FROM dashboard_entity ) as dashboardCount, " + + "(SELECT COUNT(fqnHash) FROM pipeline_entity ) as pipelineCount, " + + "(SELECT COUNT(fqnHash) FROM ml_model_entity ) as mlmodelCount, " + + "(SELECT COUNT(fqnHash) FROM storage_container_entity ) as storageContainerCount, " + + "(SELECT COUNT(fqnHash) FROM search_index_entity ) as searchIndexCount, " + + "(SELECT COUNT(nameHash) FROM glossary_entity ) as glossaryCount, " + + "(SELECT COUNT(fqnHash) FROM glossary_term_entity ) as glossaryTermCount, " + + "(SELECT (SELECT COUNT(nameHash) FROM metadata_service_entity ) + " + + "(SELECT COUNT(nameHash) FROM dbservice_entity )+" + + "(SELECT COUNT(nameHash) FROM messaging_service_entity )+ " + + "(SELECT COUNT(nameHash) FROM dashboard_service_entity )+ " + + "(SELECT COUNT(nameHash) FROM pipeline_service_entity )+ " + + "(SELECT COUNT(nameHash) FROM mlmodel_service_entity )+ " + + "(SELECT COUNT(nameHash) FROM search_service_entity )+ " + + "(SELECT COUNT(nameHash) FROM storage_service_entity )) as servicesCount, " + + "(SELECT COUNT(nameHash) FROM user_entity AND (JSON_EXTRACT(json, '$.isBot') IS NULL OR JSON_EXTRACT(json, '$.isBot') = FALSE)) as userCount, " + + "(SELECT COUNT(nameHash) FROM team_entity ) as teamCount, " + + "(SELECT COUNT(fqnHash) FROM test_suite ) as testSuiteCount", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT (SELECT COUNT(*) FROM table_entity ) as tableCount, " + + "(SELECT COUNT(*) FROM topic_entity ) as topicCount, " + + "(SELECT COUNT(*) FROM dashboard_entity ) as dashboardCount, " + + "(SELECT COUNT(*) FROM pipeline_entity ) as pipelineCount, " + + "(SELECT COUNT(*) FROM ml_model_entity ) as mlmodelCount, " + + "(SELECT COUNT(*) FROM storage_container_entity ) as storageContainerCount, " + + "(SELECT COUNT(*) FROM search_index_entity ) as searchIndexCount, " + + "(SELECT COUNT(*) FROM glossary_entity ) as glossaryCount, " + + "(SELECT COUNT(*) FROM glossary_term_entity ) as glossaryTermCount, " + + "(SELECT (SELECT COUNT(*) FROM metadata_service_entity ) + " + + "(SELECT COUNT(*) FROM dbservice_entity )+" + + "(SELECT COUNT(*) FROM messaging_service_entity )+ " + + "(SELECT COUNT(*) FROM dashboard_service_entity )+ " + + "(SELECT COUNT(*) FROM pipeline_service_entity )+ " + + "(SELECT COUNT(*) FROM mlmodel_service_entity )+ " + + "(SELECT COUNT(*) FROM search_service_entity )+ " + + "(SELECT COUNT(*) FROM storage_service_entity )) as servicesCount, " + + "(SELECT COUNT(*) FROM user_entity AND (json#>'{isBot}' IS NULL OR ((json#>'{isBot}')::boolean) = FALSE)) as userCount, " + + "(SELECT COUNT(*) FROM team_entity ) as teamCount, " + + "(SELECT COUNT(*) FROM test_suite ) as testSuiteCount", + connectionType = POSTGRES) + @RegisterRowMapper(CollectionDAO.EntitiesCountRowMapper.class) + EntitiesCount getAggregatedEntitiesCount(@Define("cond") String cond) throws StatementException; + + @ConnectionAwareSqlQuery( + value = + "SELECT (SELECT COUNT(nameHash) FROM dbservice_entity ) as databaseServiceCount, " + + "(SELECT COUNT(nameHash) FROM messaging_service_entity ) as messagingServiceCount, " + + "(SELECT COUNT(nameHash) FROM dashboard_service_entity ) as dashboardServiceCount, " + + "(SELECT COUNT(nameHash) FROM pipeline_service_entity ) as pipelineServiceCount, " + + "(SELECT COUNT(nameHash) FROM mlmodel_service_entity ) as mlModelServiceCount, " + + "(SELECT COUNT(nameHash) FROM storage_service_entity ) as storageServiceCount, " + + "(SELECT COUNT(nameHash) FROM search_service_entity ) as searchServiceCount", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT (SELECT COUNT(*) FROM dbservice_entity ) as databaseServiceCount, " + + "(SELECT COUNT(*) FROM messaging_service_entity ) as messagingServiceCount, " + + "(SELECT COUNT(*) FROM dashboard_service_entity ) as dashboardServiceCount, " + + "(SELECT COUNT(*) FROM pipeline_service_entity ) as pipelineServiceCount, " + + "(SELECT COUNT(*) FROM mlmodel_service_entity ) as mlModelServiceCount, " + + "(SELECT COUNT(*) FROM storage_service_entity ) as storageServiceCount, " + + "(SELECT COUNT(*) FROM search_service_entity ) as searchServiceCount", + connectionType = POSTGRES) + @RegisterRowMapper(CollectionDAO.ServicesCountRowMapper.class) + ServicesCount getAggregatedServicesCount(@Define("cond") String cond) throws StatementException; + + @SqlQuery("SELECT configType,json FROM openmetadata_settings") + @RegisterRowMapper(SettingsRowMapper.class) + List getAllConfig() throws StatementException; + + @SqlQuery("SELECT configType, json FROM openmetadata_settings WHERE configType = :configType") + @RegisterRowMapper(SettingsRowMapper.class) + Settings getConfigWithKey(@Bind("configType") String configType) throws StatementException; + + @ConnectionAwareSqlUpdate( + value = + "INSERT into openmetadata_settings (configType, json)" + + "VALUES (:configType, :json) ON DUPLICATE KEY UPDATE json = :json", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT into openmetadata_settings (configType, json)" + + "VALUES (:configType, :json :: jsonb) ON CONFLICT (configType) DO UPDATE SET json = EXCLUDED.json", + connectionType = POSTGRES) + void insertSettings(@Bind("configType") String configType, @Bind("json") String json); + + @SqlUpdate(value = "DELETE from openmetadata_settings WHERE configType = :configType") + void delete(@Bind("configType") String configType); + + @SqlQuery("SELECT 42") + Integer testConnection() throws StatementException; + + @ConnectionAwareSqlQuery( + value = + "SELECT JSON_EXTRACT(json, '$.fullyQualifiedName') FROM

WHERE id NOT IN ( SELECT toId FROM entity_relationship WHERE fromEntity = :fromEntity AND toEntity = :toEntity)", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json ->> 'fullyQualifiedName' FROM
WHERE id NOT IN ( SELECT toId FROM entity_relationship WHERE fromEntity = :fromEntity AND toEntity = :toEntity)", + connectionType = POSTGRES) + List getBrokenRelationFromParentToChild( + @Define("table") String tableName, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity); + + @SqlUpdate( + value = + "DELETE FROM
WHERE id NOT IN (SELECT toId FROM entity_relationship WHERE fromEntity = :fromEntity AND toEntity = :toEntity)") + int deleteBrokenRelationFromParentToChild( + @Define("table") String tableName, + @Bind("fromEntity") String fromEntity, + @Bind("toEntity") String toEntity); + } + + class SettingsRowMapper implements RowMapper { + @Override + public Settings map(ResultSet rs, StatementContext ctx) throws SQLException { + return getSettings(SettingsType.fromValue(rs.getString("configType")), rs.getString("json")); + } + + public static Settings getSettings(SettingsType configType, String json) { + Settings settings = new Settings(); + settings.setConfigType(configType); + Object value = + switch (configType) { + case EMAIL_CONFIGURATION -> JsonUtils.readValue(json, SmtpSettings.class); + case OPEN_METADATA_BASE_URL_CONFIGURATION -> JsonUtils.readValue( + json, OpenMetadataBaseUrlConfiguration.class); + case CUSTOM_UI_THEME_PREFERENCE -> JsonUtils.readValue(json, UiThemePreference.class); + case LOGIN_CONFIGURATION -> JsonUtils.readValue(json, LoginConfiguration.class); + case SLACK_APP_CONFIGURATION, SLACK_INSTALLER, SLACK_BOT, SLACK_STATE -> JsonUtils + .readValue(json, String.class); + case PROFILER_CONFIGURATION -> JsonUtils.readValue(json, ProfilerConfiguration.class); + case SEARCH_SETTINGS -> JsonUtils.readValue(json, SearchSettings.class); + case ASSET_CERTIFICATION_SETTINGS -> JsonUtils.readValue( + json, AssetCertificationSettings.class); + case WORKFLOW_SETTINGS -> JsonUtils.readValue(json, WorkflowSettings.class); + case LINEAGE_SETTINGS -> JsonUtils.readValue(json, LineageSettings.class); + case AUTHENTICATION_CONFIGURATION -> JsonUtils.readValue( + json, AuthenticationConfiguration.class); + case AUTHORIZER_CONFIGURATION -> JsonUtils.readValue( + json, AuthorizerConfiguration.class); + case ENTITY_RULES_SETTINGS -> JsonUtils.readValue(json, EntityRulesSettings.class); + case SCIM_CONFIGURATION -> JsonUtils.readValue(json, ScimConfiguration.class); + case OPEN_LINEAGE_SETTINGS -> JsonUtils.readValue(json, OpenLineageSettings.class); + case TEAMS_APP_CONFIGURATION -> JsonUtils.readValue(json, TeamsAppConfiguration.class); + case MCP_CONFIGURATION -> JsonUtils.readValue(json, MCPConfiguration.class); + case GLOSSARY_TERM_RELATION_SETTINGS -> JsonUtils.readValue( + json, GlossaryTermRelationSettings.class); + default -> throw new IllegalArgumentException("Invalid Settings Type " + configType); + }; + settings.setConfigValue(value); + return settings; + } + } + + class TokenRowMapper implements RowMapper { + @Override + public TokenInterface map(ResultSet rs, StatementContext ctx) throws SQLException { + return getToken(TokenType.fromValue(rs.getString("tokenType")), rs.getString("json")); + } + + public static TokenInterface getToken(TokenType type, String json) { + return switch (type) { + case EMAIL_VERIFICATION -> JsonUtils.readValue(json, EmailVerificationToken.class); + case PASSWORD_RESET -> JsonUtils.readValue(json, PasswordResetToken.class); + case REFRESH_TOKEN -> JsonUtils.readValue(json, RefreshToken.class); + case PERSONAL_ACCESS_TOKEN -> JsonUtils.readValue(json, PersonalAccessToken.class); + case SUPPORT_TOKEN -> JsonUtils.readValue(json, SupportToken.class); + }; + } + } + + class UserSessionRowMapper implements RowMapper { + @Override + public UserSession map(ResultSet rs, StatementContext ctx) throws SQLException { + return JsonUtils.readValue(rs.getString("json"), UserSession.class); + } + } + + // OAuth 2.0 Row Mappers + class OAuthClientRowMapper implements RowMapper { + @Override + public OAuthRecords.OAuthClientRecord map(ResultSet rs, StatementContext ctx) + throws SQLException { + return new OAuthRecords.OAuthClientRecord( + UUID.fromString(rs.getString("id")), + rs.getString("client_id"), + rs.getString("client_secret_encrypted"), + rs.getString("client_name"), + JsonUtils.readObjects(rs.getString("redirect_uris"), String.class), + JsonUtils.readObjects(rs.getString("grant_types"), String.class), + rs.getString("token_endpoint_auth_method"), + JsonUtils.readObjects(rs.getString("scopes"), String.class)); + } + } + + class OAuthAuthorizationCodeRowMapper + implements RowMapper { + @Override + public OAuthRecords.OAuthAuthorizationCodeRecord map(ResultSet rs, StatementContext ctx) + throws SQLException { + return new OAuthRecords.OAuthAuthorizationCodeRecord( + rs.getString("code"), + rs.getString("client_id"), + rs.getString("user_name"), + rs.getString("code_challenge"), + rs.getString("code_challenge_method"), + rs.getString("redirect_uri"), + JsonUtils.readObjects(rs.getString("scopes"), String.class), + rs.getLong("expires_at"), + rs.getBoolean("used")); + } + } + + class OAuthAccessTokenRowMapper implements RowMapper { + @Override + public OAuthRecords.OAuthAccessTokenRecord map(ResultSet rs, StatementContext ctx) + throws SQLException { + return new OAuthRecords.OAuthAccessTokenRecord( + UUID.fromString(rs.getString("id")), + rs.getString("token_hash"), + rs.getString("access_token_encrypted"), + rs.getString("client_id"), + rs.getString("user_name"), + JsonUtils.readObjects(rs.getString("scopes"), String.class), + rs.getLong("expires_at")); + } + } + + class OAuthRefreshTokenRowMapper implements RowMapper { + @Override + public OAuthRecords.OAuthRefreshTokenRecord map(ResultSet rs, StatementContext ctx) + throws SQLException { + return new OAuthRecords.OAuthRefreshTokenRecord( + UUID.fromString(rs.getString("id")), + rs.getString("token_hash"), + rs.getString("refresh_token_encrypted"), + rs.getString("client_id"), + rs.getString("user_name"), + JsonUtils.readObjects(rs.getString("scopes"), String.class), + rs.getLong("expires_at"), + rs.getBoolean("revoked")); + } + } + + interface TokenDAO { + @SqlQuery("SELECT tokenType, json FROM user_tokens WHERE token = :token") + @RegisterRowMapper(TokenRowMapper.class) + TokenInterface findByToken(@Bind("token") String token) throws StatementException; + + @SqlQuery( + "SELECT tokenType, json FROM user_tokens WHERE userId = :userId AND tokenType = :tokenType ") + @RegisterRowMapper(TokenRowMapper.class) + List getAllUserTokenWithType( + @BindUUID("userId") UUID userId, @Bind("tokenType") String tokenType) + throws StatementException; + + @ConnectionAwareSqlUpdate( + value = "INSERT INTO user_tokens (json) VALUES (:json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO user_tokens (json) VALUES (:json :: jsonb)", + connectionType = POSTGRES) + void insert(@Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "UPDATE user_tokens SET json = :json WHERE token = :token", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE user_tokens SET json = (:json :: jsonb) WHERE token = :token", + connectionType = POSTGRES) + void update(@Bind("token") String token, @Bind("json") String json); + + @SqlUpdate(value = "DELETE from user_tokens WHERE token = :token") + void delete(@Bind("token") String token); + + @SqlUpdate(value = "DELETE from user_tokens WHERE token IN ()") + void deleteAll(@BindList("tokenIds") List tokens); + + @SqlUpdate(value = "DELETE from user_tokens WHERE userid = :userid AND tokenType = :tokenType") + void deleteTokenByUserAndType( + @BindUUID("userid") UUID userid, @Bind("tokenType") String tokenType); + } + + interface UserSessionDAO { + @SqlQuery("SELECT json FROM user_session WHERE id = :id") + @RegisterRowMapper(UserSessionRowMapper.class) + UserSession findById(@Bind("id") String id) throws StatementException; + + @SqlQuery( + "SELECT json FROM user_session WHERE userId = :userId AND status = :status " + + "ORDER BY COALESCE(lastAccessedAt, 0) ASC LIMIT :limit") + @RegisterRowMapper(UserSessionRowMapper.class) + List findByUserIdAndStatus( + @Bind("userId") String userId, @Bind("status") String status, @Bind("limit") int limit) + throws StatementException; + + @SqlQuery( + "SELECT json FROM user_session " + + "WHERE status IN () AND expiresAt <= :now " + + "ORDER BY updatedAt ASC LIMIT :limit") + @RegisterRowMapper(UserSessionRowMapper.class) + List findSessionsExpiredByAbsoluteTimeout( + @BindList("statuses") List statuses, + @Bind("now") long now, + @Bind("limit") int limit) + throws StatementException; + + @SqlQuery( + "SELECT json FROM user_session " + + "WHERE status IN () AND idleExpiresAt <= :now " + + "ORDER BY updatedAt ASC LIMIT :limit") + @RegisterRowMapper(UserSessionRowMapper.class) + List findSessionsExpiredByIdleTimeout( + @BindList("statuses") List statuses, + @Bind("now") long now, + @Bind("limit") int limit) + throws StatementException; + + @SqlQuery( + "SELECT json FROM user_session WHERE status IN () AND updatedAt <= :cutoff " + + "ORDER BY updatedAt ASC LIMIT :limit") + @RegisterRowMapper(UserSessionRowMapper.class) + List findSessionsToPrune( + @BindList("statuses") List statuses, + @Bind("cutoff") long cutoff, + @Bind("limit") int limit) + throws StatementException; + + @ConnectionAwareSqlUpdate( + value = "INSERT INTO user_session (json) VALUES (:json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO user_session (json) VALUES (:json :: jsonb)", + connectionType = POSTGRES) + void insert(@Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE user_session SET json = :json WHERE id = :id AND version = :expectedVersion", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE user_session SET json = (:json :: jsonb) WHERE id = :id AND version = :expectedVersion", + connectionType = POSTGRES) + int updateIfVersion( + @Bind("id") String id, + @Bind("expectedVersion") long expectedVersion, + @Bind("json") String json); + + @SqlUpdate("DELETE FROM user_session WHERE id = :id") + void delete(@Bind("id") String id); + + @SqlUpdate("DELETE FROM user_session WHERE id IN ()") + int deleteByIds(@BindList("ids") List ids); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 51dd5dc1ecab..4f5424fae7d0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -118,7 +118,7 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.ExtensionRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.ExtensionRecord; import org.openmetadata.service.jdbi3.FeedRepository.TaskWorkflow; import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext; import org.openmetadata.service.resources.databases.DatabaseUtil; @@ -1449,7 +1449,8 @@ public Table addDataModel(UUID tableId, DataModel dataModel) { // addDataModel bypasses the EntityRepository.update() path, so invalidateCachesAfterStore // never runs. Drop every cached variant manually so the next GET rebuilds with the freshly // merged tags/dataModel instead of stale pre-merge JSON. - invalidateCacheForEntity(entityType, table.getId(), table.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + entityType, table.getId(), table.getFullyQualifiedName()); setFieldsInternal(table, new Fields(Set.of(FIELD_OWNERS), FIELD_OWNERS)); setFieldsInternal(table, new Fields(Set.of(FIELD_TAGS), FIELD_TAGS)); return table; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java index c0d2d505a7e5..5f863aee0b47 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java @@ -72,7 +72,7 @@ import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.governance.workflows.WorkflowHandler; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.jdbi3.FeedRepository.TaskWorkflow; import org.openmetadata.service.jdbi3.FeedRepository.ThreadContext; import org.openmetadata.service.resources.feeds.MessageParser; @@ -998,12 +998,13 @@ public void updateNameAndParent(Tag updated) { // Capture the descendants so the post-write pass can re-evict any entry a racing reader // re-populated with the pre-rename row between this call and tagDAO.updateFqn below. // The pass below runs after updateFqn but inside this transaction — see - // EntityRepository.invalidateCacheForRenameCascade for the residual pre-commit window. + // EntityCacheInvalidator.invalidateCacheForRenameCascade for the residual pre-commit + // window. List renamedTags = - invalidateCacheForRenameCascade(Entity.TAG, oldFqn); + EntityCacheInvalidator.invalidateCacheForRenameCascade(Entity.TAG, oldFqn); // Drop cached entity JSON / bundle for every entity tagged with this tag (or any // descendant). Done BEFORE the DB rename so the search lookup still matches by old FQN. - invalidateCacheForTaggedEntitiesAndDescendants(Entity.TAG, oldFqn); + EntityCacheInvalidator.invalidateCacheForTaggedEntitiesAndDescendants(Entity.TAG, oldFqn); daoCollection.tagDAO().updateFqn(oldFqn, newFqn); daoCollection.tagUsageDAO().rename(TagSource.CLASSIFICATION.ordinal(), oldFqn, newFqn); @@ -1018,7 +1019,7 @@ public void updateNameAndParent(Tag updated) { PolicyConditionUpdater.renamePrefixInCondition( condition, oldFqn, newFqn, PolicyConditionUpdater.TAG_FUNCTIONS)); - finishInvalidateCacheForRenameCascade(Entity.TAG, renamedTags); + EntityCacheInvalidator.finishInvalidateCacheForRenameCascade(Entity.TAG, renamedTags); } if (classificationChanged) { @@ -1088,7 +1089,7 @@ private void invalidateTags(UUID tagId) { // The name of the tag changed. Invalidate that tag and all the children from the cache List tagRecords = findToRecords(tagId, TAG, Relationship.CONTAINS, TAG); - CACHE_WITH_ID.invalidate(new ImmutablePair<>(TAG, tagId)); + EntityCaches.CACHE_WITH_ID.invalidate(new ImmutablePair<>(TAG, tagId)); for (EntityRelationshipRecord tagRecord : tagRecords) { invalidateTags(tagRecord.getId()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java index a23862635616..319cc26f7255 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java @@ -45,9 +45,11 @@ import jakarta.ws.rs.core.Response; import java.io.IOException; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -91,7 +93,6 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; import org.openmetadata.service.resources.feeds.FeedUtil; import org.openmetadata.service.resources.teams.TeamResource; import org.openmetadata.service.search.DefaultInheritedFieldEntitySearch; @@ -327,14 +328,19 @@ private void fetchAndSetChildrenCount(List teams, Fields fields) { if (!fields.contains("childrenCount") || teams == null || teams.isEmpty()) { return; } + List nonOrgTeamIds = new ArrayList<>(); for (Team team : teams) { - if (organization != null && team.getId().equals(organization.getId())) { - List children = daoCollection.teamDAO().listTeamsUnderOrganization(team.getId()); - team.setChildrenCount(children.size()); + if (isOrganizationTeam(team)) { + team.setChildrenCount( + daoCollection.teamDAO().listTeamsUnderOrganization(team.getId()).size()); } else { - // For other teams, count direct children - List children = findTo(team.getId(), TEAM, Relationship.PARENT_OF, TEAM); - team.setChildrenCount(children != null ? children.size() : 0); + nonOrgTeamIds.add(team.getId().toString()); + } + } + Map> childrenByParent = batchChildTeamsByParent(nonOrgTeamIds); + for (Team team : teams) { + if (!isOrganizationTeam(team)) { + team.setChildrenCount(childrenByParent.getOrDefault(team.getId(), List.of()).size()); } } } @@ -343,16 +349,122 @@ private void fetchAndSetUserCount(List teams, Fields fields) { if (!fields.contains("userCount") || teams == null || teams.isEmpty()) { return; } - + List rootIds = teams.stream().map(Team::getId).distinct().collect(Collectors.toList()); + Map> childrenMap = new HashMap<>(); + Set subtreeTeamIds = discoverSubtreeTeams(rootIds, childrenMap); + Map> directUsers = batchFetchDirectUsers(subtreeTeamIds); for (Team team : teams) { - List userIds = new ArrayList<>(); - List userRecordList = getUsersRelationshipRecords(team.getId()); - for (EntityRelationshipRecord userRecord : userRecordList) { - userIds.add(userRecord.getId().toString()); + team.setUserCount(countSubtreeUsers(team.getId(), childrenMap, directUsers)); + } + } + + private boolean isOrganizationTeam(Team team) { + return organization != null && team.getId().equals(organization.getId()); + } + + private Set discoverSubtreeTeams(List rootIds, Map> childrenMap) { + Set visited = new HashSet<>(); + List frontier = new ArrayList<>(rootIds); + while (!frontier.isEmpty()) { + visited.addAll(frontier); + Map> levelChildren = fetchChildTeams(frontier); + childrenMap.putAll(levelChildren); + frontier = nextFrontier(levelChildren, visited); + } + return visited; + } + + private List nextFrontier(Map> levelChildren, Set visited) { + return levelChildren.values().stream() + .flatMap(List::stream) + .distinct() + .filter(id -> !visited.contains(id)) + .collect(Collectors.toList()); + } + + private Map> fetchChildTeams(List teamIds) { + Map> result = new HashMap<>(); + List nonOrgIds = new ArrayList<>(); + for (UUID teamId : teamIds) { + if (organization != null && teamId.equals(organization.getId())) { + result.put( + teamId, + EntityUtil.strToIds(daoCollection.teamDAO().listTeamsUnderOrganization(teamId))); + } else { + nonOrgIds.add(teamId.toString()); } - Set userIdsSet = new HashSet<>(userIds); - team.setUserCount(userIdsSet.size()); } + result.putAll(batchChildTeamsByParent(nonOrgIds)); + return result; + } + + private Map> batchChildTeamsByParent(List parentIds) { + Map> result = new HashMap<>(); + if (nullOrEmpty(parentIds)) { + return result; + } + List records = + daoCollection + .relationshipDAO() + .findToBatch(parentIds, TEAM, TEAM, Relationship.PARENT_OF.ordinal(), ALL); + Set liveChildren = nonDeletedTeamIds(records); + for (CollectionDAO.EntityRelationshipObject record : records) { + UUID childId = UUID.fromString(record.getToId()); + if (liveChildren.contains(childId)) { + result + .computeIfAbsent(UUID.fromString(record.getFromId()), k -> new ArrayList<>()) + .add(childId); + } + } + return result; + } + + private Set nonDeletedTeamIds(List records) { + List childIds = + records.stream() + .map(r -> UUID.fromString(r.getToId())) + .distinct() + .collect(Collectors.toList()); + Set live = new HashSet<>(); + if (!childIds.isEmpty()) { + Entity.getEntityReferencesByIds(TEAM, childIds, NON_DELETED) + .forEach(ref -> live.add(ref.getId())); + } + return live; + } + + private Map> batchFetchDirectUsers(Set teamIds) { + Map> directUsers = new HashMap<>(); + if (nullOrEmpty(teamIds)) { + return directUsers; + } + List ids = teamIds.stream().map(UUID::toString).collect(Collectors.toList()); + List records = + daoCollection + .relationshipDAO() + .findToBatch(ids, TEAM, Entity.USER, Relationship.HAS.ordinal(), ALL); + for (CollectionDAO.EntityRelationshipObject record : records) { + directUsers + .computeIfAbsent(UUID.fromString(record.getFromId()), k -> new HashSet<>()) + .add(UUID.fromString(record.getToId())); + } + return directUsers; + } + + private int countSubtreeUsers( + UUID rootId, Map> childrenMap, Map> directUsers) { + Set users = new HashSet<>(); + Set visited = new HashSet<>(); + Deque stack = new ArrayDeque<>(); + stack.push(rootId); + while (!stack.isEmpty()) { + UUID teamId = stack.pop(); + if (visited.add(teamId)) { + users.addAll(directUsers.getOrDefault(teamId, Set.of())); + childrenMap.getOrDefault(teamId, List.of()).forEach(stack::push); + } + } + return users.size(); } private void fetchAndSetOwns(List teams, Fields fields) { @@ -751,26 +863,11 @@ private List getUsers(Team team) { return findTo(team.getId(), TEAM, Relationship.HAS, Entity.USER); } - private List getUsersRelationshipRecords(UUID teamId) { - List userRecord = - findToRecords(teamId, TEAM, Relationship.HAS, Entity.USER); - List children = getChildren(teamId); - for (EntityReference child : children) { - userRecord.addAll(getUsersRelationshipRecords(child.getId())); - } - return userRecord; - } - private Integer getUserCount(UUID teamId) { - List userIds = new ArrayList<>(); - List userRecordList = getUsersRelationshipRecords(teamId); - for (EntityRelationshipRecord userRecord : userRecordList) { - userIds.add(userRecord.getId().toString()); - } - Set userIdsSet = new HashSet<>(userIds); - userIds.clear(); - userIds.addAll(userIdsSet); - return userIds.size(); + Map> childrenMap = new HashMap<>(); + Set subtreeTeamIds = discoverSubtreeTeams(List.of(teamId), childrenMap); + Map> directUsers = batchFetchDirectUsers(subtreeTeamIds); + return countSubtreeUsers(teamId, childrenMap, directUsers); } private List getOwns(Team team) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java new file mode 100644 index 000000000000..20dc89cf4a9d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java @@ -0,0 +1,1880 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.schema.type.Relationship.CONTAINS; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import lombok.Getter; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindList; +import org.jdbi.v3.sqlobject.customizer.BindMap; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.jdbi.v3.sqlobject.statement.UseRowMapper; +import org.openmetadata.schema.analytics.WebAnalyticEvent; +import org.openmetadata.schema.dataInsight.DataInsightChart; +import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.TestDefinition; +import org.openmetadata.schema.tests.TestSuite; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.util.jdbi.BindFQN; +import org.openmetadata.service.util.jdbi.BindJsonContains; +import org.openmetadata.service.util.jdbi.BindListFQN; + +public interface TimeSeriesDAOs { + @CreateSqlObject + TestDefinitionDAO testDefinitionDAO(); + + @CreateSqlObject + TestSuiteDAO testSuiteDAO(); + + @CreateSqlObject + TestCaseDAO testCaseDAO(); + + @CreateSqlObject + WebAnalyticEventDAO webAnalyticEventDAO(); + + @CreateSqlObject + DataInsightCustomChartDAO dataInsightCustomChartDAO(); + + @CreateSqlObject + DataInsightChartDAO dataInsightChartDAO(); + + @CreateSqlObject + EntityExtensionTimeSeriesDAO entityExtensionTimeSeriesDao(); + + @CreateSqlObject + AppsDataStore appStoreDAO(); + + @CreateSqlObject + AppExtensionTimeSeries appExtensionTimeSeriesDao(); + + @CreateSqlObject + ReportDataTimeSeriesDAO reportDataTimeSeriesDao(); + + @CreateSqlObject + ProfilerDataTimeSeriesDAO profilerDataTimeSeriesDao(); + + @CreateSqlObject + DataQualityDataTimeSeriesDAO dataQualityDataTimeSeriesDao(); + + @CreateSqlObject + QueryCostTimeSeriesDAO queryCostRecordTimeSeriesDAO(); + + @CreateSqlObject + TestCaseResolutionStatusTimeSeriesDAO testCaseResolutionStatusTimeSeriesDao(); + + @CreateSqlObject + TestCaseResultTimeSeriesDAO testCaseResultTimeSeriesDao(); + + @CreateSqlObject + TestCaseDimensionResultTimeSeriesDAO testCaseDimensionResultTimeSeriesDao(); + + interface TestDefinitionDAO extends EntityDAO { + @Override + default String getTableName() { + return "test_definition"; + } + + @Override + default Class getEntityClass() { + return TestDefinition.class; + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String entityType = filter.getQueryParam("entityType"); + String testPlatform = filter.getQueryParam("testPlatform"); + String supportedDataType = filter.getQueryParam("supportedDataType"); + String supportedService = filter.getQueryParam("supportedService"); + String enabled = filter.getQueryParam("enabled"); + String condition = filter.getCondition(); + + if (entityType == null + && testPlatform == null + && supportedDataType == null + && supportedService == null + && enabled == null) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + + StringBuilder mysqlCondition = new StringBuilder(); + StringBuilder psqlCondition = new StringBuilder(); + + mysqlCondition.append(String.format("%s ", condition)); + psqlCondition.append(String.format("%s ", condition)); + + if (testPlatform != null) { + filter.queryParams.put("testPlatformLike", String.format("%%%s%%", testPlatform)); + mysqlCondition.append("AND json_extract(json, '$.testPlatforms') LIKE :testPlatformLike "); + psqlCondition.append("AND json->>'testPlatforms' LIKE :testPlatformLike "); + } + + if (entityType != null) { + mysqlCondition.append("AND entityType=:entityType "); + psqlCondition.append("AND entityType=:entityType "); + } + + if (supportedDataType != null) { + filter.queryParams.put("supportedDataTypeExact", supportedDataType); + mysqlCondition.append( + "AND JSON_CONTAINS(json, JSON_QUOTE(:supportedDataTypeExact), '$.supportedDataTypes') "); + psqlCondition.append( + "AND json->'supportedDataTypes' @> to_jsonb(CAST(:supportedDataTypeExact AS TEXT)) "); + } + + if (supportedService != null) { + filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); + mysqlCondition.append( + "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " + + "OR json_extract(json, '$.supportedServices') IS NULL " + + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); + psqlCondition.append( + "AND (json->>'supportedServices' = '[]' " + + "OR json->>'supportedServices' IS NULL " + + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); + } + + if (enabled != null) { + String enabledValue = Boolean.parseBoolean(enabled) ? "TRUE" : "FALSE"; + mysqlCondition.append("AND enabled=").append(enabledValue).append(" "); + psqlCondition.append("AND enabled=").append(enabledValue).append(" "); + } + + return listBefore( + getTableName(), + filter.getQueryParams(), + mysqlCondition.toString(), + psqlCondition.toString(), + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String entityType = filter.getQueryParam("entityType"); + String testPlatform = filter.getQueryParam("testPlatform"); + String supportedDataType = filter.getQueryParam("supportedDataType"); + String supportedService = filter.getQueryParam("supportedService"); + String enabled = filter.getQueryParam("enabled"); + String condition = filter.getCondition(); + + if (entityType == null + && testPlatform == null + && supportedDataType == null + && supportedService == null + && enabled == null) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + + StringBuilder mysqlCondition = new StringBuilder(); + StringBuilder psqlCondition = new StringBuilder(); + + mysqlCondition.append(String.format("%s ", condition)); + psqlCondition.append(String.format("%s ", condition)); + + if (testPlatform != null) { + filter.queryParams.put("testPlatformLike", String.format("%%%s%%", testPlatform)); + mysqlCondition.append("AND json_extract(json, '$.testPlatforms') LIKE :testPlatformLike "); + psqlCondition.append("AND json->>'testPlatforms' LIKE :testPlatformLike "); + } + + if (entityType != null) { + mysqlCondition.append("AND entityType = :entityType "); + psqlCondition.append("AND entityType = :entityType "); + } + + if (supportedDataType != null) { + filter.queryParams.put("supportedDataTypeExact", supportedDataType); + mysqlCondition.append( + "AND JSON_CONTAINS(json, JSON_QUOTE(:supportedDataTypeExact), '$.supportedDataTypes') "); + psqlCondition.append( + "AND json->'supportedDataTypes' @> to_jsonb(CAST(:supportedDataTypeExact AS TEXT)) "); + } + + if (supportedService != null) { + filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); + mysqlCondition.append( + "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " + + "OR json_extract(json, '$.supportedServices') IS NULL " + + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); + psqlCondition.append( + "AND (json->>'supportedServices' = '[]' " + + "OR json->>'supportedServices' IS NULL " + + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); + } + + if (enabled != null) { + String enabledValue = Boolean.parseBoolean(enabled) ? "TRUE" : "FALSE"; + mysqlCondition.append("AND enabled=").append(enabledValue).append(" "); + psqlCondition.append("AND enabled=").append(enabledValue).append(" "); + } + + return listAfter( + getTableName(), + filter.getQueryParams(), + mysqlCondition.toString(), + psqlCondition.toString(), + limit, + afterName, + afterId); + } + + @Override + default int listCount(ListFilter filter) { + String entityType = filter.getQueryParam("entityType"); + String testPlatform = filter.getQueryParam("testPlatform"); + String supportedDataType = filter.getQueryParam("supportedDataType"); + String supportedService = filter.getQueryParam("supportedService"); + String enabled = filter.getQueryParam("enabled"); + String condition = filter.getCondition(); + + if (entityType == null + && testPlatform == null + && supportedDataType == null + && supportedService == null + && enabled == null) { + return EntityDAO.super.listCount(filter); + } + + StringBuilder mysqlCondition = new StringBuilder(); + StringBuilder psqlCondition = new StringBuilder(); + + mysqlCondition.append(String.format("%s ", condition)); + psqlCondition.append(String.format("%s ", condition)); + + if (testPlatform != null) { + filter.queryParams.put("testPlatformLike", String.format("%%%s%%", testPlatform)); + mysqlCondition.append("AND json_extract(json, '$.testPlatforms') LIKE :testPlatformLike "); + psqlCondition.append("AND json->>'testPlatforms' LIKE :testPlatformLike "); + } + + if (entityType != null) { + mysqlCondition.append("AND entityType=:entityType "); + psqlCondition.append("AND entityType=:entityType "); + } + + if (supportedDataType != null) { + filter.queryParams.put("supportedDataTypeExact", supportedDataType); + mysqlCondition.append( + "AND JSON_CONTAINS(json, JSON_QUOTE(:supportedDataTypeExact), '$.supportedDataTypes') "); + psqlCondition.append( + "AND json->'supportedDataTypes' @> to_jsonb(CAST(:supportedDataTypeExact AS TEXT)) "); + } + + if (supportedService != null) { + filter.queryParams.put("supportedServiceLike", String.format("%%%s%%", supportedService)); + mysqlCondition.append( + "AND (json_extract(json, '$.supportedServices') = JSON_ARRAY() " + + "OR json_extract(json, '$.supportedServices') IS NULL " + + "OR json_extract(json, '$.supportedServices') LIKE :supportedServiceLike) "); + psqlCondition.append( + "AND (json->>'supportedServices' = '[]' " + + "OR json->>'supportedServices' IS NULL " + + "OR json->>'supportedServices' LIKE :supportedServiceLike) "); + } + + if (enabled != null) { + String enabledValue = Boolean.parseBoolean(enabled) ? "TRUE" : "FALSE"; + mysqlCondition.append("AND enabled=").append(enabledValue).append(" "); + psqlCondition.append("AND enabled=").append(enabledValue).append(" "); + } + + return listCount( + getTableName(), + filter.getQueryParams(), + getNameHashColumn(), + mysqlCondition.toString(), + psqlCondition.toString()); + } + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT name, id, json FROM
AND " + + "(
.name < :beforeName OR (
.name = :beforeName AND
.id < :beforeId)) " + + "ORDER BY name DESC,id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT name, id, json FROM
AND " + + "(
.name < :beforeName OR (
.name = :beforeName AND
.id < :beforeId)) " + + "ORDER BY name DESC,id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id", + connectionType = POSTGRES) + List listBefore( + @Define("table") String table, + @BindMap Map params, + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit", + connectionType = POSTGRES) + List listAfter( + @Define("table") String table, + @BindMap Map params, + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId); + + @ConnectionAwareSqlQuery( + value = "SELECT count() FROM
", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM
", + connectionType = POSTGRES) + int listCount( + @Define("table") String table, + @BindMap Map params, + @Define("nameHashColumn") String nameHashColumn, + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond); + } + + interface TestSuiteDAO extends EntityDAO { + @Override + default String getTableName() { + return "test_suite"; + } + + @Override + default Class getEntityClass() { + return TestSuite.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default int listCount(ListFilter filter) { + String mySqlCondition = filter.getCondition(getTableName()); + String postgresCondition = filter.getCondition(getTableName()); + boolean includeEmptyTestSuite = + Boolean.parseBoolean(filter.getQueryParam("includeEmptyTestSuites")); + if (!includeEmptyTestSuite) { + String condition = + String.format( + "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", + getTableName(), CONTAINS.ordinal(), Entity.TEST_CASE); + mySqlCondition = condition; + postgresCondition = condition; + + mySqlCondition = + String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); + postgresCondition = + String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); + } + return listCountDistinct( + getTableName(), + mySqlCondition, + postgresCondition, + String.format("%s.%s", getTableName(), getNameHashColumn())); + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String mySqlCondition = filter.getCondition(getTableName()); + String postgresCondition = filter.getCondition(getTableName()); + String groupBy = ""; + boolean includeEmptyTestSuite = + Boolean.parseBoolean(filter.getQueryParam("includeEmptyTestSuites")); + if (!includeEmptyTestSuite) { + groupBy = + String.format( + "group by %s.json, %s.name, %s.id", getTableName(), getTableName(), getTableName()); + String condition = + String.format( + "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", + getTableName(), CONTAINS.ordinal(), Entity.TEST_CASE); + mySqlCondition = condition; + postgresCondition = condition; + mySqlCondition = + String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); + postgresCondition = + String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); + } + return listBefore( + getTableName(), mySqlCondition, postgresCondition, limit, beforeName, beforeId, groupBy); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String mySqlCondition = filter.getCondition(getTableName()); + String postgresCondition = filter.getCondition(getTableName()); + String groupBy = ""; + boolean includeEmptyTestSuite = + Boolean.parseBoolean(filter.getQueryParam("includeEmptyTestSuites")); + if (!includeEmptyTestSuite) { + groupBy = + String.format( + "group by %s.json, %s.name, %s.id", getTableName(), getTableName(), getTableName()); + String condition = + String.format( + "INNER JOIN entity_relationship er ON %s.id=er.fromId AND er.relation=%s AND er.toEntity='%s'", + getTableName(), CONTAINS.ordinal(), Entity.TEST_CASE); + mySqlCondition = condition; + postgresCondition = condition; + + mySqlCondition = + String.format("%s %s", mySqlCondition, filter.getCondition(getTableName())); + postgresCondition = + String.format("%s %s", postgresCondition, filter.getCondition(getTableName())); + } + return listAfter( + getTableName(), mySqlCondition, postgresCondition, limit, afterName, afterId, groupBy); + } + + @SqlQuery( + "SELECT json FROM
tn\n" + + "INNER JOIN (SELECT DISTINCT fromId FROM entity_relationship er\n" + + " AND toEntity = 'testSuite' and fromEntity = :entityType) er ON fromId = tn.id\n" + + "LIMIT :limit OFFSET :offset;") + List listEntitiesWithTestSuite( + @Define("table") String table, + @BindMap Map params, + @Define("cond") String cond, + @Bind("entityType") String entityType, + @Bind("limit") int limit, + @Bind("offset") int offset); + + default List listEntitiesWithTestsuite( + ListFilter filter, String table, String entityType, int limit, int offset) { + return listEntitiesWithTestSuite( + table, filter.getQueryParams(), filter.getCondition(), entityType, limit, offset); + } + + @SqlQuery( + "SELECT COUNT(DISTINCT fromId) FROM entity_relationship er\n" + + " AND toEntity = 'testSuite' and fromEntity = :entityType;") + Integer countEntitiesWithTestSuite( + @BindMap Map params, + @Define("cond") String cond, + @Bind("entityType") String entityType); + + default Integer countEntitiesWithTestsuite(ListFilter filter, String entityType) { + return countEntitiesWithTestSuite(filter.getQueryParams(), filter.getCondition(), entityType); + } + } + + interface TestCaseDAO extends EntityDAO { + @Override + default String getTableName() { + return "test_case"; + } + + @Override + default Class getEntityClass() { + return TestCase.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + default int countOfTestCases(List testCaseIds) { + return countOfTestCases(getTableName(), testCaseIds.stream().map(Object::toString).toList()); + } + + @SqlQuery("SELECT count(*) FROM
WHERE id IN ()") + int countOfTestCases( + @Define("table") String table, @BindList("testCaseIds") List testCaseIds); + + /** + * Returns ids of test cases whose entityFQN equals {@code entityFQN} (table-level tests) or + * starts with {@code entityFQNPrefix} (column-level tests). The prefix must already have LIKE + * metacharacters escaped — callers should route through + * {@link org.openmetadata.service.util.LikeEscape#escape(String)} and append {@code ".%"}. + * Uses {@code ESCAPE '!'} to match the convention used elsewhere in this DAO; backslash is + * unsafe (MySQL treats it as a string-literal escape and JDBI's ColonPrefixSqlParser + * mishandles literal {@code '\'} inside single-quoted SQL strings). + */ + @SqlQuery( + "SELECT id FROM test_case WHERE entityFQN = :entityFQN " + + "OR entityFQN LIKE :entityFQNPrefix ESCAPE '!'") + List findIdsByEntityFQN( + @Bind("entityFQN") String entityFQN, @Bind("entityFQNPrefix") String entityFQNPrefix); + + class TestCaseRecord { + @Getter String json; + @Getter Integer rank; + + public TestCaseRecord(String json, Integer rank) { + this.json = json; + this.rank = rank; + } + } + + class TestCaseRecordMapper implements RowMapper { + @Override + public TestCaseRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new TestCaseRecord(rs.getString("json"), rs.getInt("ranked")); + } + } + } + + interface WebAnalyticEventDAO extends EntityDAO { + @Override + default String getTableName() { + return "web_analytic_event"; + } + + @Override + default Class getEntityClass() { + return WebAnalyticEvent.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface DataInsightCustomChartDAO extends EntityDAO { + @Override + default String getTableName() { + return "di_chart_entity"; + } + + @Override + default Class getEntityClass() { + return DataInsightCustomChart.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface DataInsightChartDAO extends EntityDAO { + @Override + default String getTableName() { + return "data_insight_chart"; + } + + @Override + default Class getEntityClass() { + return DataInsightChart.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface EntityExtensionTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "entity_extension_time_series"; + } + + @ConnectionAwareSqlQuery( + value = + "SELECT " + + " DATE(FROM_UNIXTIME(eets.timestamp / 1000)) as date_key, " + + " JSON_UNQUOTE(JSON_EXTRACT(eets.json, '$.executionStatus')) as status, " + + " COUNT(*) as count " + + "FROM entity_extension_time_series eets " + + "INNER JOIN pipeline_entity pe ON eets.entityFQNHash = pe.fqnHash " + + "WHERE eets.extension = 'pipeline.pipelineStatus' " + + " AND pe.deleted = 0 " + + " AND eets.timestamp >= :startTs " + + " AND eets.timestamp <= :endTs " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + "GROUP BY date_key, status " + + "ORDER BY date_key ASC", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT " + + " DATE(TO_TIMESTAMP(eets.timestamp / 1000)) as date_key, " + + " eets.json->>'executionStatus' as status, " + + " COUNT(*) as count " + + "FROM entity_extension_time_series eets " + + "INNER JOIN pipeline_entity pe ON eets.entityFQNHash = pe.fqnHash " + + "WHERE eets.extension = 'pipeline.pipelineStatus' " + + " AND pe.deleted = false " + + " AND eets.timestamp >= :startTs " + + " AND eets.timestamp <= :endTs " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + "GROUP BY date_key, status " + + "ORDER BY date_key ASC", + connectionType = POSTGRES) + @RegisterRowMapper(CollectionDAO.ExecutionTrendRowMapper.class) + List getExecutionTrendData( + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs, + @Define("pipelineFqnFilter") String pipelineFqnFilter, + @Define("serviceTypeFilter") String serviceTypeFilter, + @Define("serviceFilter") String serviceFilter, + @Define("mysqlStatusFilter") String mysqlStatusFilter, + @Define("postgresStatusFilter") String postgresStatusFilter, + @Define("domainFilter") String domainFilter, + @Define("ownerFilter") String ownerFilter, + @Define("tierFilter") String tierFilter); + + @ConnectionAwareSqlQuery( + value = + "WITH runtime_calc AS ( " + + " SELECT " + + " eets.*, " + + " pe.fqnHash, " + + " CASE " + + " WHEN JSON_LENGTH(JSON_EXTRACT(eets.json, '$.taskStatus')) > 0 " + + " AND JSON_EXTRACT(eets.json, '$.taskStatus[0].endTime') IS NOT NULL THEN " + + " ( " + + " SELECT MAX(CAST(JSON_EXTRACT(task.value, '$.endTime') AS UNSIGNED)) " + + " FROM JSON_TABLE(eets.json, '$.taskStatus[*]' COLUMNS(value JSON PATH '$')) AS task " + + " WHERE JSON_EXTRACT(task.value, '$.endTime') IS NOT NULL " + + " ) - ( " + + " SELECT MIN(CAST(JSON_EXTRACT(task.value, '$.startTime') AS UNSIGNED)) " + + " FROM JSON_TABLE(eets.json, '$.taskStatus[*]' COLUMNS(value JSON PATH '$')) AS task " + + " WHERE JSON_EXTRACT(task.value, '$.startTime') IS NOT NULL " + + " ) " + + " WHEN JSON_EXTRACT(eets.json, '$.endTime') IS NOT NULL THEN " + + " JSON_EXTRACT(eets.json, '$.endTime') - eets.timestamp " + + " ELSE NULL " + + " END AS runtime " + + " FROM entity_extension_time_series eets " + + " INNER JOIN pipeline_entity pe ON eets.entityFQNHash = pe.fqnHash " + + " WHERE eets.extension = 'pipeline.pipelineStatus' " + + " AND pe.deleted = 0 " + + " AND eets.timestamp >= :startTs " + + " AND eets.timestamp <= :endTs " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + ") " + + "SELECT " + + " DATE(FROM_UNIXTIME(timestamp / 1000)) as date_key, " + + " MIN(timestamp) as first_timestamp, " + + " MAX(runtime) as max_runtime, " + + " MIN(runtime) as min_runtime, " + + " AVG(runtime) as avg_runtime, " + + " COUNT(DISTINCT fqnHash) as total_pipelines " + + "FROM runtime_calc " + + "WHERE runtime IS NOT NULL " + + "GROUP BY date_key " + + "ORDER BY date_key ASC", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "WITH runtime_calc AS ( " + + " SELECT " + + " eets.timestamp, " + + " eets.json, " + + " pe.fqnHash, " + + " CASE " + + " WHEN jsonb_array_length(COALESCE(eets.json->'taskStatus', '[]'::jsonb)) > 0 " + + " AND EXISTS ( " + + " SELECT 1 FROM jsonb_array_elements(eets.json->'taskStatus') AS task " + + " WHERE task->>'endTime' IS NOT NULL " + + " ) THEN " + + " ( " + + " SELECT MAX((task->>'endTime')::bigint) " + + " FROM jsonb_array_elements(eets.json->'taskStatus') AS task " + + " WHERE task->>'endTime' IS NOT NULL " + + " ) - ( " + + " SELECT MIN((task->>'startTime')::bigint) " + + " FROM jsonb_array_elements(eets.json->'taskStatus') AS task " + + " WHERE task->>'startTime' IS NOT NULL " + + " ) " + + " WHEN eets.json->>'endTime' IS NOT NULL THEN " + + " (eets.json->>'endTime')::bigint - eets.timestamp " + + " ELSE NULL " + + " END AS runtime " + + " FROM entity_extension_time_series eets " + + " INNER JOIN pipeline_entity pe ON eets.entityFQNHash = pe.fqnHash " + + " WHERE eets.extension = 'pipeline.pipelineStatus' " + + " AND pe.deleted = false " + + " AND eets.timestamp >= :startTs " + + " AND eets.timestamp <= :endTs " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + ") " + + "SELECT " + + " DATE(TO_TIMESTAMP(timestamp / 1000)) as date_key, " + + " MIN(timestamp) as first_timestamp, " + + " MAX(runtime) as max_runtime, " + + " MIN(runtime) as min_runtime, " + + " AVG(runtime) as avg_runtime, " + + " COUNT(DISTINCT fqnHash) as total_pipelines " + + "FROM runtime_calc " + + "WHERE runtime IS NOT NULL " + + "GROUP BY date_key " + + "ORDER BY date_key ASC", + connectionType = POSTGRES) + @RegisterRowMapper(CollectionDAO.RuntimeTrendRowMapper.class) + List getRuntimeTrendData( + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs, + @Define("pipelineFqnFilter") String pipelineFqnFilter, + @Define("serviceTypeFilter") String serviceTypeFilter, + @Define("serviceFilter") String serviceFilter, + @Define("mysqlStatusFilter") String mysqlStatusFilter, + @Define("postgresStatusFilter") String postgresStatusFilter, + @Define("domainFilter") String domainFilter, + @Define("ownerFilter") String ownerFilter, + @Define("tierFilter") String tierFilter); + + @ConnectionAwareSqlQuery( + value = + "SELECT " + + " JSON_UNQUOTE(JSON_EXTRACT(pe.json, '$.serviceType')) as service_type, " + + " COUNT(*) as pipeline_count " + + "FROM pipeline_entity pe " + + "LEFT JOIN entity_extension_time_series eets " + + " ON pe.fqnHash = eets.entityFQNHash " + + " AND eets.extension = 'pipeline.pipelineStatus' " + + "WHERE pe.deleted = 0 " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + "GROUP BY service_type", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT " + + " pe.json->>'serviceType' as service_type, " + + " COUNT(*) as pipeline_count " + + "FROM pipeline_entity pe " + + "LEFT JOIN entity_extension_time_series eets " + + " ON pe.fqnHash = eets.entityFQNHash " + + " AND eets.extension = 'pipeline.pipelineStatus' " + + "WHERE pe.deleted = false " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + "GROUP BY service_type", + connectionType = POSTGRES) + @RegisterRowMapper(CollectionDAO.ServiceBreakdownRowMapper.class) + List getServiceBreakdown( + @Define("serviceTypeFilter") String serviceTypeFilter, + @Define("serviceFilter") String serviceFilter, + @Define("mysqlStatusFilter") String mysqlStatusFilter, + @Define("postgresStatusFilter") String postgresStatusFilter, + @Define("domainFilter") String domainFilter, + @Define("ownerFilter") String ownerFilter, + @Define("tierFilter") String tierFilter, + @Define("startTsFilter") String startTsFilter, + @Define("endTsFilter") String endTsFilter); + + @ConnectionAwareSqlQuery( + value = + "SELECT " + + " COUNT(DISTINCT pe.fqnHash) as total_pipelines, " + + " COUNT(DISTINCT CASE WHEN eets.entityFQNHash IS NOT NULL THEN pe.fqnHash END) as active_pipelines, " + + " COUNT(DISTINCT CASE WHEN JSON_UNQUOTE(JSON_EXTRACT(eets.json, '$.executionStatus')) = 'Successful' THEN pe.fqnHash END) as successful_pipelines, " + + " COUNT(DISTINCT CASE WHEN JSON_UNQUOTE(JSON_EXTRACT(eets.json, '$.executionStatus')) = 'Failed' THEN pe.fqnHash END) as failed_pipelines " + + "FROM pipeline_entity pe " + + "LEFT JOIN entity_extension_time_series eets " + + " ON pe.fqnHash = eets.entityFQNHash " + + " AND eets.extension = 'pipeline.pipelineStatus' " + + "WHERE pe.deleted = 0 " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + " ", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT " + + " COUNT(DISTINCT pe.fqnHash) as total_pipelines, " + + " COUNT(DISTINCT CASE WHEN eets.entityFQNHash IS NOT NULL THEN pe.fqnHash END) as active_pipelines, " + + " COUNT(DISTINCT CASE WHEN eets.json->>'executionStatus' = 'Successful' THEN pe.fqnHash END) as successful_pipelines, " + + " COUNT(DISTINCT CASE WHEN eets.json->>'executionStatus' = 'Failed' THEN pe.fqnHash END) as failed_pipelines " + + "FROM pipeline_entity pe " + + "LEFT JOIN entity_extension_time_series eets " + + " ON pe.fqnHash = eets.entityFQNHash " + + " AND eets.extension = 'pipeline.pipelineStatus' " + + "WHERE pe.deleted = false " + + " " + + " " + + " " + + " " + + " " + + " " + + " " + + " ", + connectionType = POSTGRES) + @RegisterRowMapper(CollectionDAO.PipelineMetricsRowMapper.class) + CollectionDAO.PipelineMetricsRow getPipelineMetricsData( + @Define("serviceTypeFilter") String serviceTypeFilter, + @Define("serviceFilter") String serviceFilter, + @Define("mysqlStatusFilter") String mysqlStatusFilter, + @Define("postgresStatusFilter") String postgresStatusFilter, + @Define("domainFilter") String domainFilter, + @Define("ownerFilter") String ownerFilter, + @Define("tierFilter") String tierFilter, + @Define("startTsFilter") String startTsFilter, + @Define("endTsFilter") String endTsFilter); + + @ConnectionAwareSqlQuery( + value = + "SELECT pe.id, pe.json, " + + "(SELECT eets_inner.json FROM entity_extension_time_series eets_inner " + + " WHERE eets_inner.entityFQNHash = pe.fqnHash " + + " AND eets_inner.extension = 'pipeline.pipelineStatus' " + + " ORDER BY eets_inner.timestamp DESC LIMIT 1) as latest_status " + + "FROM pipeline_entity pe " + + "WHERE pe.deleted = 0 " + + " " + + " " + + " " + + " " + + " " + + " AND (:search IS NULL OR pe.name LIKE CONCAT('%', :search, '%') OR JSON_UNQUOTE(JSON_EXTRACT(pe.json, '$.fullyQualifiedName')) LIKE CONCAT('%', :search, '%')) " + + " " + + "ORDER BY pe.name " + + "LIMIT :limit OFFSET :offset", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT pe.id, pe.json, " + + "(SELECT eets_inner.json FROM entity_extension_time_series eets_inner " + + " WHERE eets_inner.entityFQNHash = pe.fqnHash " + + " AND eets_inner.extension = 'pipeline.pipelineStatus' " + + " ORDER BY eets_inner.timestamp DESC LIMIT 1) as latest_status " + + "FROM pipeline_entity pe " + + "WHERE pe.deleted = false " + + " " + + " " + + " " + + " " + + " " + + " AND (:search IS NULL OR pe.name LIKE '%' || :search || '%' OR pe.json->>'fullyQualifiedName' LIKE '%' || :search || '%') " + + " " + + "ORDER BY pe.name " + + "LIMIT :limit OFFSET :offset", + connectionType = POSTGRES) + @RegisterRowMapper(CollectionDAO.PipelineSummaryRowMapper.class) + List listPipelineSummariesFiltered( + @Define("serviceFilter") String serviceFilter, + @Define("mysqlServiceTypeFilter") String mysqlServiceTypeFilter, + @Define("postgresServiceTypeFilter") String postgresServiceTypeFilter, + @Define("domainFilter") String domainFilter, + @Define("ownerFilter") String ownerFilter, + @Define("tierFilter") String tierFilter, + @Define("mysqlStatusFilter") String mysqlStatusFilter, + @Define("postgresStatusFilter") String postgresStatusFilter, + @Bind("search") String search, + @Bind("limit") int limit, + @Bind("offset") int offset); + + @ConnectionAwareSqlQuery( + value = + "SELECT COUNT(DISTINCT pe.id) " + + "FROM pipeline_entity pe " + + "WHERE pe.deleted = 0 " + + " " + + " " + + " " + + " " + + " " + + " AND (:search IS NULL OR pe.name LIKE CONCAT('%', :search, '%') OR JSON_UNQUOTE(JSON_EXTRACT(pe.json, '$.fullyQualifiedName')) LIKE CONCAT('%', :search, '%')) " + + " ", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT COUNT(DISTINCT pe.id) " + + "FROM pipeline_entity pe " + + "WHERE pe.deleted = false " + + " " + + " " + + " " + + " " + + " " + + " AND (:search IS NULL OR pe.name LIKE '%' || :search || '%' OR pe.json->>'fullyQualifiedName' LIKE '%' || :search || '%') " + + " ", + connectionType = POSTGRES) + int countPipelineSummariesFiltered( + @Define("serviceFilter") String serviceFilter, + @Define("mysqlServiceTypeFilter") String mysqlServiceTypeFilter, + @Define("postgresServiceTypeFilter") String postgresServiceTypeFilter, + @Define("domainFilter") String domainFilter, + @Define("ownerFilter") String ownerFilter, + @Define("tierFilter") String tierFilter, + @Define("mysqlStatusFilter") String mysqlStatusFilter, + @Define("postgresStatusFilter") String postgresStatusFilter, + @Bind("search") String search); + } + + interface AppsDataStore { + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json) ON DUPLICATE KEY UPDATE json = VALUES(json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json :: jsonb) ON CONFLICT (identifier, type) DO UPDATE SET json = EXCLUDED.json", + connectionType = POSTGRES) + void insert( + @Bind("identifier") String identifier, + @Bind("type") String type, + @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_data_store set json = :json where identifier = :identifier AND type=:type", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_data_store set json = (:json :: jsonb) where identifier = :identifier AND type=:type", + connectionType = POSTGRES) + void update( + @Bind("identifier") String identifier, + @Bind("type") String type, + @Bind("json") String json); + + @SqlUpdate("DELETE FROM apps_data_store WHERE identifier = :identifier AND type = :type") + void delete(@Bind("identifier") String identifier, @Bind("type") String type); + + @SqlQuery( + "SELECT count(*) FROM apps_data_store where identifier = :identifier AND type = :type") + int listAppDataCount(@Bind("identifier") String identifier, @Bind("type") String type); + + @SqlQuery( + "SELECT json FROM apps_data_store where identifier in () AND type = :type") + List listAppsDataWithIds( + @BindList("identifier") List identifier, @Bind("type") String type); + + @SqlQuery("SELECT json FROM apps_data_store where type = :type") + List listAppsDataWithType(@Bind("type") String type); + + @SqlQuery("SELECT json FROM apps_data_store where identifier = :identifier AND type = :type") + String findAppData(@Bind("identifier") String identifier, @Bind("type") String type); + } + + interface AppExtensionTimeSeries { + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO apps_extension_time_series(json, extension) VALUES (:json, :extension)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO apps_extension_time_series(json, extension) VALUES (:json :: jsonb, :extension)", + connectionType = POSTGRES) + void insert(@Bind("json") String json, @Bind("extension") String extension); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'stopped') where appId=:appId AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status'", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"stopped\"') WHERE appId = :appId AND json->>'status' = 'running' AND extension = 'status'", + connectionType = POSTGRES) + void markStaleEntriesStopped(@Bind("appId") String appId); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'stopped') WHERE appName=:appName AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status'", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"stopped\"') WHERE appName = :appName AND json->>'status' = 'running' AND extension = 'status'", + connectionType = POSTGRES) + void markStaleEntriesStoppedByName(@Bind("appName") String appName); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'stopped') WHERE appName=:appName AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status' AND timestamp < :beforeTimestamp", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"stopped\"') WHERE appName = :appName AND json->>'status' = 'running' AND extension = 'status' AND timestamp < :beforeTimestamp", + connectionType = POSTGRES) + void markStaleEntriesStoppedBefore( + @Bind("appName") String appName, @Bind("beforeTimestamp") long beforeTimestamp); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'failed') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status'", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"failed\"') WHERE json->>'status' = 'running' AND extension = 'status'", + connectionType = POSTGRES) + void markAllStaleEntriesFailed(); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'failed') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status' AND appName != :appName", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"failed\"') WHERE json->>'status' = 'running' AND extension = 'status' AND appName != :appName", + connectionType = POSTGRES) + void markAllStaleEntriesFailedExcludingApp(@Bind("appName") String appName); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'failed') WHERE appName=:appName AND JSON_UNQUOTE(JSON_EXTRACT(json, '$.status')) = 'running' AND extension = 'status'", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"failed\"') WHERE appName = :appName AND json->>'status' = 'running' AND extension = 'status'", + connectionType = POSTGRES) + void markRunningEntriesFailedByName(@Bind("appName") String appName); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.status', 'running') WHERE appId = :appId AND extension = 'status' AND timestamp = :timestamp", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{status}', '\"running\"') WHERE appId = :appId AND extension = 'status' AND timestamp = :timestamp", + connectionType = POSTGRES) + void markEntryRunning(@Bind("appId") String appId, @Bind("timestamp") long timestamp); + + @SqlQuery( + "SELECT json FROM apps_extension_time_series WHERE appId = :appId AND extension = :extension AND timestamp = :timestamp") + String getByAppIdAndTimestamp( + @Bind("appId") String appId, + @Bind("timestamp") long timestamp, + @Bind("extension") String extension); + + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series set json = :json where appId=:appId and timestamp=:timestamp and extension=:extension", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "UPDATE apps_extension_time_series set json = (:json :: jsonb) where appId=:appId and timestamp=:timestamp and extension=:extension", + connectionType = POSTGRES) + void update( + @Bind("appId") String appId, + @Bind("json") String json, + @Bind("timestamp") Long timestamp, + @Bind("extension") String extension); + + @SqlUpdate( + "DELETE FROM apps_extension_time_series WHERE appId = :appId AND extension = :extension") + void delete(@Bind("appId") String appId, @Bind("extension") String extension); + + @SqlUpdate("DELETE FROM apps_extension_time_series WHERE appId = :appId") + void deleteAllByAppId(@Bind("appId") String appId); + + @SqlQuery( + "SELECT count(*) FROM apps_extension_time_series where appId = :appId and extension = :extension AND ") + int listAppExtensionCount( + @Bind("appId") String appId, + @Bind("extension") String extension, + @BindJsonContains(value = "service_filter", path = "$.services", property = "id") + UUID service); + + @SqlQuery( + "SELECT count(*) FROM apps_extension_time_series where appId = :appId and extension = :extension AND timestamp > :startTime AND ") + int listAppExtensionCountAfterTime( + @Bind("appId") String appId, + @Bind("startTime") long startTime, + @Bind("extension") String extension, + @BindJsonContains( + value = "service_filter", + path = "$.services", + property = "id", + ifNull = "TRUE") + UUID service); + + @SqlQuery( + "SELECT json FROM apps_extension_time_series where appId = :appId AND extension = :extension AND ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + List listAppExtension( + @Bind("appId") String appId, + @Bind("limit") int limit, + @Bind("offset") int offset, + @Bind("extension") String extension, + @BindJsonContains( + value = "service_filter", + path = "$.services", + property = "id", + ifNull = "TRUE") + UUID service); + + @SqlQuery( + "SELECT json FROM apps_extension_time_series where appId = :appId AND extension = :extension AND timestamp > :startTime AND ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + List listAppExtensionAfterTime( + @Bind("appId") String appId, + @Bind("limit") int limit, + @Bind("offset") int offset, + @Bind("startTime") long startTime, + @Bind("extension") String extension, + @BindJsonContains( + value = "service_filter", + path = "$.services", + property = "id", + ifNull = "TRUE") + UUID service); + + // Prepare methods to get extension by name instead of ID + // For example, for limits we need to fetch by app name to ensure if we reinstall the app, + // they'll still be taken into account + @SqlQuery( + "SELECT count(*) FROM apps_extension_time_series where appName = :appName and extension = :extension") + int listAppExtensionCountByName( + @Bind("appName") String appName, @Bind("extension") String extension); + + @SqlQuery( + "SELECT count(*) FROM apps_extension_time_series where appName = :appName and extension = :extension AND timestamp > :startTime") + int listAppExtensionCountAfterTimeByName( + @Bind("appName") String appName, + @Bind("startTime") long startTime, + @Bind("extension") String extension); + + @SqlQuery( + "SELECT json FROM apps_extension_time_series where appName = :appName AND extension = :extension ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + List listAppExtensionByName( + @Bind("appName") String appName, + @Bind("limit") int limit, + @Bind("offset") int offset, + @Bind("extension") String extension); + + @SqlQuery( + "SELECT json FROM apps_extension_time_series where appName = :appName AND extension = :extension AND timestamp > :startTime ORDER BY timestamp DESC LIMIT :limit OFFSET :offset") + List listAppExtensionAfterTimeByName( + @Bind("appName") String appName, + @Bind("limit") int limit, + @Bind("offset") int offset, + @Bind("startTime") long startTime, + @Bind("extension") String extension); + + @SqlQuery( + "SELECT json FROM apps_extension_time_series where appName = :appName AND extension = :extension AND timestamp >= :startTime AND timestamp < :endTime ORDER BY timestamp ASC LIMIT :limit OFFSET :offset") + List listAppExtensionInWindowByName( + @Bind("appName") String appName, + @Bind("limit") int limit, + @Bind("offset") int offset, + @Bind("startTime") long startTime, + @Bind("endTime") long endTime, + @Bind("extension") String extension); + + default List listAppExtensionAfterTime( + String appId, int limit, int offset, long startTime, String extension) { + return listAppExtensionAfterTime(appId, limit, offset, startTime, extension, null); + } + + default int listAppExtensionCountAfterTime(String appName, long startTime, String extension) { + return listAppExtensionCountAfterTime(appName, startTime, extension, null); + } + + default List listAppExtension(String appName, int limit, int offset, String extension) { + return listAppExtension(appName, limit, offset, extension, null); + } + } + + interface ReportDataTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "report_data_time_series"; + } + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType and date = :date", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType and DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) = DATE(:date)", + connectionType = POSTGRES) + void deleteReportDataTypeAtDate( + @BindFQN("reportDataType") String reportDataType, @Bind("date") String date); + + @SqlUpdate("DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType") + void deletePreviousReportData(@BindFQN("reportDataType") String reportDataType); + } + + interface ProfilerDataTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "profiler_data_time_series"; + } + + @SqlQuery( + "SELECT p.entityFQNHash, p.json " + + "FROM
p " + + "JOIN (" + + " SELECT entityFQNHash, MAX(timestamp) AS latestTs " + + " FROM
" + + " WHERE entityFQNHash IN () AND extension = :extension " + + " GROUP BY entityFQNHash" + + ") latest " + + "ON p.entityFQNHash = latest.entityFQNHash AND p.timestamp = latest.latestTs " + + "WHERE p.extension = :extension " + + "AND p.entityFQNHash IN ()") + @RegisterRowMapper(LatestExtensionRecordMapper.class) + List getLatestExtensionsBatch( + @Define("table") String table, + @BindListFQN("entityFQNHashes") List entityFQNHashes, + @Bind("extension") String extension); + + default List getLatestExtensionsBatch( + List entityFQNHashes, String extension) { + if (entityFQNHashes == null || entityFQNHashes.isEmpty()) { + return Collections.emptyList(); + } + return getLatestExtensionsBatch(getTimeSeriesTableName(), entityFQNHashes, extension); + } + + @SqlQuery( + "SELECT json FROM
" + + "AND timestamp >= :startTs and timestamp <= :endTs ORDER BY timestamp DESC") + List listEntityProfileAtTimestamp( + @Define("table") String table, + @BindMap Map params, + @Define("cond") String cond, + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs); + + default List listEntityProfileData(ListFilter filter, Long startTs, Long endTs) { + return listEntityProfileAtTimestamp( + getTimeSeriesTableName(), filter.getQueryParams(), filter.getCondition(), startTs, endTs); + } + + @SqlUpdate("DELETE FROM
AND timestamp = :timestamp") + void deleteEntityProfileData( + @Define("table") String table, + @BindMap Map params, + @Define("cond") String cond, + @Bind("timestamp") Long timestamp); + + default void deleteEntityProfileData(ListFilter filter, Long timestamp) { + deleteEntityProfileData( + getTimeSeriesTableName(), filter.getQueryParams(), filter.getCondition(), timestamp); + } + + // profiler_data_time_series has no id column (unique key is + // entityFQNHash + extension + operation + timestamp), so we limit by + // row count using single-table DELETE+LIMIT on MySQL and ctid IN (...) on Postgres. + // This bounds the rows deleted per batch, matching the other orphan-cleanup queries. + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM profiler_data_time_series " + + "WHERE NOT EXISTS (" + + " SELECT 1 FROM table_entity te " + + " WHERE te.fqnHash = profiler_data_time_series.entityFQNHash" + + ") " + + "LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM profiler_data_time_series " + + "WHERE ctid IN (" + + " SELECT pdts.ctid FROM profiler_data_time_series pdts " + + " WHERE NOT EXISTS (" + + " SELECT 1 FROM table_entity te " + + " WHERE te.fqnHash = pdts.entityFQNHash" + + " ) " + + " LIMIT :limit" + + ")", + connectionType = POSTGRES) + int deleteOrphanedRecords(@Bind("limit") int limit); + + record LatestExtensionRecord(String entityFQNHash, String json) {} + + class LatestExtensionRecordMapper implements RowMapper { + @Override + public LatestExtensionRecord map(ResultSet rs, StatementContext ctx) throws SQLException { + return new LatestExtensionRecord(rs.getString("entityFQNHash"), rs.getString("json")); + } + } + } + + interface DataQualityDataTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "data_quality_data_time_series"; + } + + @RegisterRowMapper(LatestRecordWithFQNHashMapper.class) + @SqlQuery( + "SELECT t1.entityFQNHash, t1.json FROM data_quality_data_time_series t1 " + + "INNER JOIN (SELECT entityFQNHash, MAX(timestamp) as maxTs " + + "FROM data_quality_data_time_series WHERE entityFQNHash IN () " + + "GROUP BY entityFQNHash) t2 " + + "ON t1.entityFQNHash = t2.entityFQNHash AND t1.timestamp = t2.maxTs") + List getLatestRecordBatch( + @BindListFQN("entityFQNHashes") List entityFQNs); + + @SqlUpdate( + "DELETE FROM data_quality_data_time_series WHERE entityFQNHash = :testCaseFQNHash AND extension = 'testCase.testCaseResult'") + void deleteAll(@BindFQN("testCaseFQNHash") String entityFQNHash); + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) " + + "VALUES (:testCaseFQNHash, :extension, :jsonSchema, :json, :incidentStateId)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) " + + "VALUES (:testCaseFQNHash, :extension, :jsonSchema, (:json :: jsonb), :incidentStateId)", + connectionType = POSTGRES) + void insert( + @Define("table") String table, + @BindFQN("testCaseFQNHash") String testCaseFQNHash, + @Bind("extension") String extension, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json, + @Bind("incidentStateId") String incidentStateId); + + default void insert( + String entityFQNHash, + String extension, + String jsonSchema, + String json, + String incidentStateId) { + insert(getTimeSeriesTableName(), entityFQNHash, extension, jsonSchema, json, incidentStateId); + } + } + + class LatestRecordWithFQNHash { + private final String entityFQNHash; + private final String json; + + public LatestRecordWithFQNHash(String entityFQNHash, String json) { + this.entityFQNHash = entityFQNHash; + this.json = json; + } + + public String getEntityFQNHash() { + return entityFQNHash; + } + + public String getJson() { + return json; + } + } + + class LatestRecordWithFQNHashMapper implements RowMapper { + @Override + public LatestRecordWithFQNHash map(ResultSet r, StatementContext ctx) throws SQLException { + return new LatestRecordWithFQNHash(r.getString("entityFQNHash"), r.getString("json")); + } + } + + interface QueryCostTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "query_cost_time_series"; + } + + // TODO: Do not change id on override... updating json changed the id as well + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO
(entityFQNHash, jsonSchema, json) " + + "VALUES (:entityFQNHash, :jsonSchema, :json) ON DUPLICATE KEY UPDATE" + + " json = VALUES(json);", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO
(entityFQNHash, jsonSchema, json) " + + "VALUES (:entityFQNHash, :jsonSchema, (:json :: jsonb)) " + + "ON CONFLICT (entityFQNHash, timestamp) " + + "DO UPDATE SET " + + "json = EXCLUDED.json", + connectionType = POSTGRES) + void insertWithoutExtension( + @Define("table") String table, + @BindFQN("entityFQNHash") String entityFQNHash, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json); + + @SqlUpdate("DELETE FROM query_cost_time_series WHERE entityFQNHash = :entityFQNHash ") + void deleteWithEntityFqnHash(@BindFQN("entityFQNHash") String entityFQNHash); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM query_cost_time_series " + + "WHERE id IN (" + + " SELECT id FROM (" + + " SELECT qcts.id FROM query_cost_time_series qcts " + + " LEFT JOIN query_entity qe ON qcts.entityFQNHash = qe.fqnHash " + + " WHERE qe.fqnHash IS NULL " + + " LIMIT :limit" + + " ) sub" + + ")", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM query_cost_time_series " + + "WHERE id IN (" + + " SELECT qcts.id FROM query_cost_time_series qcts " + + " LEFT JOIN query_entity qe ON qcts.entityFQNHash = qe.fqnHash " + + " WHERE qe.fqnHash IS NULL " + + " LIMIT :limit" + + ")", + connectionType = POSTGRES) + int deleteOrphanedRecords(@Bind("limit") int limit); + } + + interface TestCaseResolutionStatusTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "test_case_resolution_status_time_series"; + } + + @SqlQuery( + value = + "SELECT json FROM test_case_resolution_status_time_series " + + "WHERE stateId = :stateId ORDER BY timestamp DESC") + List listTestCaseResolutionStatusesForStateId(@Bind("stateId") String stateId); + + @SqlQuery( + value = + "SELECT json FROM test_case_resolution_status_time_series " + + "WHERE entityFQNHash = :entityFQNHash ORDER BY timestamp DESC") + List listTestCaseResolutionForEntityFQNHash( + @BindFQN("entityFQNHash") String entityFqnHas); + + @SqlQuery( + value = + "SELECT json FROM test_case_resolution_status_time_series " + + "WHERE assignee = :userFqn ORDER BY timestamp DESC") + List listTestCaseResolutionForAssignee(@Bind("userFqn") String userFqn); + + @SqlQuery( + value = + "SELECT json FROM test_case_resolution_status_time_series " + + "WHERE stateId = :stateId ORDER BY timestamp ASC LIMIT 1") + String listFirstTestCaseResolutionStatusesForStateId(@Bind("stateId") String stateId); + + @SqlUpdate( + "DELETE FROM test_case_resolution_status_time_series WHERE entityFQNHash = :entityFQNHash") + void delete(@BindFQN("entityFQNHash") String entityFQNHash); + + @SqlQuery( + "SELECT json FROM " + + "(SELECT id, json, testCaseResolutionStatusType, assignee, ROW_NUMBER() OVER(PARTITION BY ORDER BY timestamp DESC) AS row_num " + + "FROM
" + + "AND timestamp BETWEEN :startTs AND :endTs " + + "ORDER BY timestamp DESC) ranked " + + " AND ranked.row_num = 1 LIMIT :limit OFFSET :offset") + List listWithOffset( + @Define("table") String table, + @BindMap Map params, + @Define("cond") String cond, + @Define("partition") String partition, + @Bind("limit") int limit, + @Bind("offset") int offset, + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs, + @BindMap Map outerParams, + @Define("outerCond") String outerFilter); + + @Override + default List listWithOffset( + ListFilter filter, int limit, int offset, Long startTs, Long endTs, boolean latest) { + if (latest) { + // When fetching latest, we need to apply Assignee and Status filters on the outer query + // i.e. after we have fetched the latest records for each testCaseFQNHash + // We'll first get the values, remove then from `filter` and then create `outerFilter` + String testCaseResolutionStatusType = filter.getQueryParam("testCaseResolutionStatusType"); + filter.removeQueryParam("testCaseResolutionStatusType"); + String assignee = filter.getQueryParam("assignee"); + filter.removeQueryParam("assignee"); + + ListFilter outerFilter = new ListFilter(null); + outerFilter.addQueryParam("testCaseResolutionStatusType", testCaseResolutionStatusType); + outerFilter.addQueryParam("assignee", assignee); + + String condition = filter.getCondition(); + condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); + + return listWithOffset( + getTimeSeriesTableName(), + filter.getQueryParams(), + condition, + getPartitionFieldName(), + limit, + offset, + startTs, + endTs, + filter.getQueryParams(), + outerFilter.getCondition()); + } + String condition = filter.getCondition(); + condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); + return listWithOffset( + getTimeSeriesTableName(), + filter.getQueryParams(), + condition, + limit, + offset, + startTs, + endTs); + } + + @Override + default int listCount(ListFilter filter, Long startTs, Long endTs, boolean latest) { + String condition = filter.getCondition(); + condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); + return latest + ? listCount( + getTimeSeriesTableName(), + getPartitionFieldName(), + filter.getQueryParams(), + condition, + startTs, + endTs) + : listCount(getTimeSeriesTableName(), filter.getQueryParams(), condition, startTs, endTs); + } + + @Override + default List listWithOffset(ListFilter filter, int limit, int offset) { + String condition = filter.getCondition(); + condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); + return listWithOffset( + getTimeSeriesTableName(), filter.getQueryParams(), condition, limit, offset); + } + + @Override + default int listCount(ListFilter filter) { + String condition = filter.getCondition(); + condition = TestCaseResolutionStatusRepository.addOriginEntityFQNJoin(filter, condition); + return listCount(getTimeSeriesTableName(), filter.getQueryParams(), condition); + } + + // relation = 9 corresponds to Relationship.PARENT_OF (the enum ordinal is stable; + // see Relationship.java where new values must be appended). The annotation can't + // reference the enum at compile time, so we inline the ordinal here. + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM test_case_resolution_status_time_series " + + "WHERE id IN (" + + " SELECT id FROM (" + + " SELECT ts.id FROM test_case_resolution_status_time_series ts " + + " LEFT JOIN entity_relationship er " + + " ON er.toId = ts.id AND er.relation = 9 " // 9 = Relationship.PARENT_OF + + " AND er.fromEntity = 'testCase' " + + " AND er.toEntity = 'testCaseResolutionStatus' " + + " WHERE er.toId IS NULL " + + " LIMIT :limit" + + " ) sub" + + ")", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM test_case_resolution_status_time_series " + + "WHERE id IN (" + + " SELECT ts.id FROM test_case_resolution_status_time_series ts " + + " LEFT JOIN entity_relationship er " + + " ON er.toId = ts.id AND er.relation = 9 " // 9 = Relationship.PARENT_OF + + " AND er.fromEntity = 'testCase' " + + " AND er.toEntity = 'testCaseResolutionStatus' " + + " WHERE er.toId IS NULL " + + " LIMIT :limit" + + ")", + connectionType = POSTGRES) + int deleteOrphanedRecords(@Bind("limit") int limit); + } + + interface TestCaseResultTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "data_quality_data_time_series"; + } + + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) " + + "VALUES (:testCaseFQNHash, :extension, :jsonSchema, :json, :incidentStateId)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO data_quality_data_time_series(entityFQNHash, extension, jsonSchema, json, incidentId) " + + "VALUES (:testCaseFQNHash, :extension, :jsonSchema, (:json :: jsonb), :incidentStateId)", + connectionType = POSTGRES) + void insert( + @Define("table") String table, + @BindFQN("testCaseFQNHash") String testCaseFQNHash, + @Bind("extension") String extension, + @Bind("jsonSchema") String jsonSchema, + @Bind("json") String json, + @Bind("incidentStateId") String incidentStateId); + + @ConnectionAwareSqlQuery( + value = + """ + SELECT dqdts1.json FROM + data_quality_data_time_series dqdts1 + INNER JOIN ( + SELECT tc.fqnHash + FROM entity_relationship er + INNER JOIN test_case tc ON er.toId = tc.id + WHERE fromEntity = 'testSuite' AND toEntity = 'testCase' AND fromId = :testSuiteId + ) ts ON dqdts1.entityFQNHash = ts.fqnHash + LEFT JOIN data_quality_data_time_series dqdts2 FORCE INDEX (idx_entity_timestamp_desc) ON + (dqdts1.entityFQNHash = dqdts2.entityFQNHash AND dqdts1.timestamp < dqdts2.timestamp) + WHERE dqdts2.entityFQNHash IS NULL""", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + """ + SELECT dqdts1.json FROM + data_quality_data_time_series dqdts1 + INNER JOIN ( + SELECT tc.fqnHash + FROM entity_relationship er + INNER JOIN test_case tc ON er.toId = tc.id + WHERE fromEntity = 'testSuite' AND toEntity = 'testCase' AND fromId = :testSuiteId + ) ts ON dqdts1.entityFQNHash = ts.fqnHash + LEFT JOIN data_quality_data_time_series dqdts2 ON + (dqdts1.entityFQNHash = dqdts2.entityFQNHash AND dqdts1.timestamp < dqdts2.timestamp) + WHERE dqdts2.entityFQNHash IS NULL""", + connectionType = POSTGRES) + List listLastTestCaseResultsForTestSuite(@BindMap Map params); + + @SqlQuery( + """ + SELECT dqdts1.json FROM + data_quality_data_time_series dqdts1 + LEFT JOIN data_quality_data_time_series dqdts2 ON + (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp) + WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""") + String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN); + + default void insert( + String testCaseFQN, + String extension, + String jsonSchema, + String json, + UUID incidentStateId) { + + insert( + getTimeSeriesTableName(), + testCaseFQN, + extension, + jsonSchema, + json, + incidentStateId != null ? incidentStateId.toString() : null); + } + + default List listLastTestCaseResultsForTestSuite(UUID testSuiteId) { + return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString())); + } + + record ResultSummaryRow( + String testSuiteId, String testCaseFQN, String testCaseStatus, long timestamp) {} + + class ResultSummaryRowMapper implements RowMapper { + @Override + public ResultSummaryRow map(ResultSet rs, StatementContext ctx) throws SQLException { + return new ResultSummaryRow( + rs.getString("testSuiteId"), + rs.getString("testCaseFQN"), + rs.getString("testCaseStatus"), + rs.getLong("timestamp")); + } + } + + @ConnectionAwareSqlQuery( + value = + """ + WITH suite_test_cases AS ( + SELECT tc.fqnHash, er.fromId as testSuiteId + FROM entity_relationship er + INNER JOIN test_case tc ON er.toId = tc.id + WHERE er.fromEntity = 'testSuite' AND er.toEntity = 'testCase' + AND er.fromId IN () + ), + latest_results AS ( + SELECT dqdts.entityFQNHash, + JSON_UNQUOTE(JSON_EXTRACT(dqdts.json, '$.testCaseFQN')) as testCaseFQN, + JSON_UNQUOTE(JSON_EXTRACT(dqdts.json, '$.testCaseStatus')) as testCaseStatus, + dqdts.timestamp, + ROW_NUMBER() OVER (PARTITION BY dqdts.entityFQNHash ORDER BY dqdts.timestamp DESC) as rn + FROM data_quality_data_time_series dqdts + WHERE dqdts.entityFQNHash IN (SELECT fqnHash FROM suite_test_cases) + ) + SELECT stc.testSuiteId, lr.testCaseFQN, lr.testCaseStatus, lr.timestamp + FROM latest_results lr + INNER JOIN suite_test_cases stc ON lr.entityFQNHash = stc.fqnHash + WHERE lr.rn = 1 + """, + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + """ + WITH suite_test_cases AS ( + SELECT tc.fqnHash, er.fromId as testSuiteId + FROM entity_relationship er + INNER JOIN test_case tc ON er.toId = tc.id + WHERE er.fromEntity = 'testSuite' AND er.toEntity = 'testCase' + AND er.fromId IN () + ), + latest_results AS ( + SELECT dqdts.entityFQNHash, + dqdts.json->>'testCaseFQN' as testCaseFQN, + dqdts.json->>'testCaseStatus' as testCaseStatus, + dqdts.timestamp, + ROW_NUMBER() OVER (PARTITION BY dqdts.entityFQNHash ORDER BY dqdts.timestamp DESC) as rn + FROM data_quality_data_time_series dqdts + WHERE dqdts.entityFQNHash IN (SELECT fqnHash FROM suite_test_cases) + ) + SELECT stc.testSuiteId, lr.testCaseFQN, lr.testCaseStatus, lr.timestamp + FROM latest_results lr + INNER JOIN suite_test_cases stc ON lr.entityFQNHash = stc.fqnHash + WHERE lr.rn = 1 + """, + connectionType = POSTGRES) + @UseRowMapper(ResultSummaryRowMapper.class) + List listResultSummariesForTestSuites( + @BindList("testSuiteIds") List testSuiteIds); + + record SuiteMaxTimestamp(String testSuiteId, long maxTimestamp) {} + + class SuiteMaxTimestampMapper implements RowMapper { + @Override + public SuiteMaxTimestamp map(ResultSet rs, StatementContext ctx) throws SQLException { + return new SuiteMaxTimestamp(rs.getString("testSuiteId"), rs.getLong("maxTimestamp")); + } + } + + @SqlQuery( + """ + SELECT er_sub.fromId as testSuiteId, MAX(dqdts.timestamp) as maxTimestamp + FROM data_quality_data_time_series dqdts + INNER JOIN ( + SELECT tc.fqnHash, er.fromId + FROM entity_relationship er + INNER JOIN test_case tc ON er.toId = tc.id + WHERE er.fromEntity = 'testSuite' AND er.toEntity = 'testCase' + AND er.fromId IN () + ) er_sub ON dqdts.entityFQNHash = er_sub.fqnHash + GROUP BY er_sub.fromId""") + @UseRowMapper(SuiteMaxTimestampMapper.class) + List getMaxTimestampForTestSuites( + @BindList("testSuiteIds") List testSuiteIds); + } + + interface TestCaseDimensionResultTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "test_case_dimension_results_time_series"; + } + + @SqlQuery( + "SELECT json FROM test_case_dimension_results_time_series " + + "WHERE entityFQNHash = :testCaseFQN AND timestamp >= :startTs AND timestamp <= :endTs " + + "ORDER BY timestamp DESC") + List listTestCaseDimensionResults( + @BindFQN("testCaseFQN") String testCaseFQN, + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs); + + @SqlQuery( + "SELECT json FROM test_case_dimension_results_time_series " + + "WHERE entityFQNHash = :testCaseFQN AND dimensionKey = :dimensionKey AND timestamp >= :startTs AND timestamp <= :endTs " + + "ORDER BY timestamp DESC") + List listTestCaseDimensionResultsByKey( + @BindFQN("testCaseFQN") String testCaseFQN, + @Bind("dimensionKey") String dimensionKey, + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs); + + @SqlQuery( + "SELECT json FROM test_case_dimension_results_time_series " + + "WHERE entityFQNHash = :testCaseFQN AND dimensionName = :dimensionName AND timestamp >= :startTs AND timestamp <= :endTs " + + "ORDER BY timestamp DESC") + List listTestCaseDimensionResultsByDimensionName( + @BindFQN("testCaseFQN") String testCaseFQN, + @Bind("dimensionName") String dimensionName, + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs); + + @SqlQuery( + "SELECT DISTINCT dimensionKey FROM test_case_dimension_results_time_series " + + "WHERE entityFQNHash = :testCaseFQN AND timestamp >= :startTs AND timestamp <= :endTs") + List listAvailableDimensionKeys( + @BindFQN("testCaseFQN") String testCaseFQN, + @Bind("startTs") Long startTs, + @Bind("endTs") Long endTs); + + @SqlUpdate( + "DELETE FROM test_case_dimension_results_time_series WHERE entityFQNHash = :testCaseFQNHash") + void deleteAll(@BindFQN("testCaseFQNHash") String testCaseFQN); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java index 3e4b2b1766a5..ff12cf1366b2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java @@ -84,8 +84,8 @@ import org.openmetadata.service.exception.BadRequestException; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.UserDAO; +import org.openmetadata.service.jdbi3.AccessControlDAOs.UserDAO; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.resources.feeds.FeedUtil; import org.openmetadata.service.resources.teams.UserResource; import org.openmetadata.service.search.DefaultInheritedFieldEntitySearch; @@ -835,12 +835,43 @@ private void fetchAndSetRoles(List users, Fields fields) { userToRoles.computeIfAbsent(userId, k -> new ArrayList<>()).add(roleRef); } + Map> userToTeams = batchFetchTeamsForUsers(userIds); + for (User user : users) { List roleRefs = userToRoles.get(user.getId()); user.setRoles(roleRefs != null ? roleRefs : new ArrayList<>()); - // Also set inherited roles - user.withInheritedRoles(getInheritedRoles(user)); + user.withInheritedRoles(getInheritedRoles(user, userToTeams.get(user.getId()))); + } + } + + private Map> batchFetchTeamsForUsers(List userIds) { + Map> userToTeams = new HashMap<>(); + List teamRecords = + daoCollection + .relationshipDAO() + .findFromBatch(userIds, Relationship.HAS.ordinal(), Entity.TEAM, Include.ALL); + for (CollectionDAO.EntityRelationshipObject record : teamRecords) { + UUID userId = UUID.fromString(record.getToId()); + EntityReference teamRef = + Entity.getEntityReferenceById( + Entity.TEAM, UUID.fromString(record.getFromId()), Include.ALL); + if (!Boolean.TRUE.equals(teamRef.getDeleted())) { + userToTeams.computeIfAbsent(userId, k -> new ArrayList<>()).add(teamRef); + } + } + return userToTeams; + } + + private List getInheritedRoles(User user, List teams) { + List roles; + if (Boolean.TRUE.equals(user.getIsBot())) { + roles = Collections.emptyList(); + } else { + List effectiveTeams = + nullOrEmpty(teams) ? new ArrayList<>(List.of(getOrganization())) : teams; + roles = SubjectContext.getRolesForTeams(effectiveTeams); } + return roles; } private void fetchAndSetOwns(List users, Fields fields) { @@ -1288,8 +1319,10 @@ private void deleteSuggestionTasksForUser(User entity) { // entries to drop. The DELETE is a direct SQL update that bypasses EntityRepository.delete // and its cache-invalidate hook — without explicit eviction the next GET on a // previously-read task returns the stale cached row even though the DB row is gone. - // FQN is required because tasks expose both GET /v1/tasks/{id} (CACHE_WITH_ID-keyed) and - // GET /v1/tasks/name/{taskId} (CACHE_WITH_NAME-keyed); dropping only by id would leave a + // FQN is required because tasks expose both GET /v1/tasks/{id} + // (EntityCaches.CACHE_WITH_ID-keyed) and + // GET /v1/tasks/name/{taskId} (EntityCaches.CACHE_WITH_NAME-keyed); dropping only by id would + // leave a // by-name reader pinned to a stale entry. List tasksToInvalidate = daoCollection.taskDAO().listIdAndFqnByCreatorAndCategory(creatorId, category); @@ -1312,7 +1345,7 @@ private void invalidateTaskCacheForIds(List tasks) { if (task.id == null) { continue; } - EntityRepository.invalidateCacheForEntity(Entity.TASK, task.id, task.fqn); + EntityCacheInvalidator.invalidateCacheForEntity(Entity.TASK, task.id, task.fqn); if (pubsub != null) { pubsub.publish(Entity.TASK, task.id, task.fqn, "bot-task-cleanup"); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java index bd5d3fa6e1ff..88bab3fd637a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java @@ -376,7 +376,8 @@ public void suspendWorkflow(WorkflowDefinition workflow) { workflow.setSuspended(true); dao.update(workflow); - invalidateCacheForEntity(entityType, workflow.getId(), workflow.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + entityType, workflow.getId(), workflow.getFullyQualifiedName()); LOG.info("Suspended workflow '{}' in Flowable engine", workflowName); } catch (IllegalArgumentException e) { // Workflow not deployed to Flowable - this can happen for workflows that haven't been @@ -400,7 +401,8 @@ public void resumeWorkflow(WorkflowDefinition workflow) { workflow.setSuspended(false); dao.update(workflow); - invalidateCacheForEntity(entityType, workflow.getId(), workflow.getFullyQualifiedName()); + EntityCacheInvalidator.invalidateCacheForEntity( + entityType, workflow.getId(), workflow.getFullyQualifiedName()); // Log the resumption LOG.info("Resumed workflow '{}' in Flowable engine", workflowName); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDocStoreDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDocStoreDAOs.java new file mode 100644 index 000000000000..6d390a7d5661 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDocStoreDAOs.java @@ -0,0 +1,686 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; +import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindMap; +import org.jdbi.v3.sqlobject.customizer.Define; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; +import org.openmetadata.schema.dataInsight.kpi.Kpi; +import org.openmetadata.schema.entities.docStore.Document; +import org.openmetadata.schema.entity.automations.Workflow; +import org.openmetadata.schema.entity.context.ContextMemory; +import org.openmetadata.schema.entity.data.APICollection; +import org.openmetadata.schema.entity.data.APIEndpoint; +import org.openmetadata.schema.entity.data.DashboardDataModel; +import org.openmetadata.schema.entity.learning.LearningResource; +import org.openmetadata.schema.governance.workflows.WorkflowDefinition; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; +import org.openmetadata.service.util.FullyQualifiedName; +import org.openmetadata.service.util.jdbi.BindFQN; +import org.openmetadata.service.util.jdbi.BindUUID; + +public interface WorkflowDocStoreDAOs { + @CreateSqlObject + KpiDAO kpiDAO(); + + @CreateSqlObject + WorkflowDAO workflowDAO(); + + @CreateSqlObject + DataModelDAO dashboardDataModelDAO(); + + @CreateSqlObject + DocStoreDAO docStoreDAO(); + + @CreateSqlObject + LearningResourceDAO learningResourceDAO(); + + @CreateSqlObject + ContextMemoryDAO contextMemoryDAO(); + + @CreateSqlObject + SuggestionDAO suggestionDAO(); + + @CreateSqlObject + APICollectionDAO apiCollectionDAO(); + + @CreateSqlObject + APIEndpointDAO apiEndpointDAO(); + + @CreateSqlObject + WorkflowDefinitionDAO workflowDefinitionDAO(); + + @CreateSqlObject + WorkflowInstanceTimeSeriesDAO workflowInstanceTimeSeriesDAO(); + + @CreateSqlObject + WorkflowInstanceStateTimeSeriesDAO workflowInstanceStateTimeSeriesDAO(); + + @CreateSqlObject + RecognizerFeedbackDAO recognizerFeedbackDAO(); + + interface KpiDAO extends EntityDAO { + @Override + default String getTableName() { + return "kpi_entity"; + } + + @Override + default Class getEntityClass() { + return Kpi.class; + } + } + + interface WorkflowDAO extends EntityDAO { + @Override + default String getTableName() { + return "automations_workflow"; + } + + @Override + default Class getEntityClass() { + return Workflow.class; + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String workflowType = filter.getQueryParam("workflowType"); + String workflowStatus = filter.getQueryParam("workflowStatus"); + String condition = filter.getCondition(); + + if (workflowType == null && workflowStatus == null) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + + StringBuilder sqlCondition = new StringBuilder(); + sqlCondition.append(String.format("%s ", condition)); + + if (workflowType != null) { + sqlCondition.append("AND workflowType=:workflowType "); + } + + if (workflowStatus != null) { + sqlCondition.append("AND status=:workflowStatus "); + } + + return listBefore( + getTableName(), + filter.getQueryParams(), + sqlCondition.toString(), + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String workflowType = filter.getQueryParam("workflowType"); + String workflowStatus = filter.getQueryParam("workflowStatus"); + String condition = filter.getCondition(); + + if (workflowType == null && workflowStatus == null) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + + StringBuilder sqlCondition = new StringBuilder(); + sqlCondition.append(String.format("%s ", condition)); + + if (workflowType != null) { + sqlCondition.append("AND workflowType=:workflowType "); + } + + if (workflowStatus != null) { + sqlCondition.append("AND status=:workflowStatus "); + } + + return listAfter( + getTableName(), + filter.getQueryParams(), + sqlCondition.toString(), + limit, + afterName, + afterId); + } + + @Override + default int listCount(ListFilter filter) { + String workflowType = filter.getQueryParam("workflowType"); + String workflowStatus = filter.getQueryParam("workflowStatus"); + String condition = filter.getCondition(); + + if (workflowType == null && workflowStatus == null) { + return EntityDAO.super.listCount(filter); + } + + StringBuilder sqlCondition = new StringBuilder(); + sqlCondition.append(String.format("%s ", condition)); + + if (workflowType != null) { + sqlCondition.append("AND workflowType=:workflowType "); + } + + if (workflowStatus != null) { + sqlCondition.append("AND status=:workflowStatus "); + } + + return listCount(getTableName(), filter.getQueryParams(), sqlCondition.toString()); + } + + @SqlQuery( + value = + "SELECT json FROM (" + + "SELECT name, id, json FROM
AND " + + "(
.name < :beforeName OR (
.name = :beforeName AND
.id < :beforeId)) " + + "ORDER BY name DESC,id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id") + List listBefore( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId); + + @SqlQuery( + value = + "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit") + List listAfter( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId); + + @SqlQuery(value = "SELECT count(*) FROM
") + int listCount( + @Define("table") String table, + @BindMap Map params, + @Define("sqlCondition") String sqlCondition); + } + + interface DataModelDAO extends EntityDAO { + @Override + default String getTableName() { + return "dashboard_data_model_entity"; + } + + @Override + default Class getEntityClass() { + return DashboardDataModel.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface DocStoreDAO extends EntityDAO { + @Override + default String getTableName() { + return "doc_store"; + } + + @Override + default Class getEntityClass() { + return Document.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + + @Override + default boolean supportsSoftDelete() { + return false; + } + + @Override + default List listBefore( + ListFilter filter, int limit, String beforeName, String beforeId) { + String entityType = filter.getQueryParam("entityType"); + String fqnPrefix = filter.getQueryParam("fqnPrefix"); + String cond = filter.getCondition(); + if (entityType == null && fqnPrefix == null) { + return EntityDAO.super.listBefore(filter, limit, beforeName, beforeId); + } + + StringBuilder mysqlCondition = new StringBuilder(); + StringBuilder psqlCondition = new StringBuilder(); + mysqlCondition.append(cond); + psqlCondition.append(cond); + + if (fqnPrefix != null) { + String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); + filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); + filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); + String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; + mysqlCondition.append(fqnCond); + psqlCondition.append(fqnCond); + } + + if (entityType != null) { + mysqlCondition.append(" AND entityType=:entityType "); + psqlCondition.append(" AND entityType=:entityType "); + } + + return listBefore( + getTableName(), + filter.getQueryParams(), + mysqlCondition.toString(), + psqlCondition.toString(), + limit, + beforeName, + beforeId); + } + + @Override + default List listAfter(ListFilter filter, int limit, String afterName, String afterId) { + String entityType = filter.getQueryParam("entityType"); + String fqnPrefix = filter.getQueryParam("fqnPrefix"); + String cond = filter.getCondition(); + + if (entityType == null && fqnPrefix == null) { + return EntityDAO.super.listAfter(filter, limit, afterName, afterId); + } + + StringBuilder mysqlCondition = new StringBuilder(); + StringBuilder psqlCondition = new StringBuilder(); + mysqlCondition.append(cond); + psqlCondition.append(cond); + + if (fqnPrefix != null) { + String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); + filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); + filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); + String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; + mysqlCondition.append(fqnCond); + psqlCondition.append(fqnCond); + } + if (entityType != null) { + mysqlCondition.append(" AND entityType=:entityType "); + psqlCondition.append(" AND entityType=:entityType "); + } + + return listAfter( + getTableName(), + filter.getQueryParams(), + mysqlCondition.toString(), + psqlCondition.toString(), + limit, + afterName, + afterId); + } + + @Override + default int listCount(ListFilter filter) { + String entityType = filter.getQueryParam("entityType"); + String fqnPrefix = filter.getQueryParam("fqnPrefix"); + String cond = filter.getCondition(); + + if (entityType == null && fqnPrefix == null) { + return EntityDAO.super.listCount(filter); + } + + StringBuilder mysqlCondition = new StringBuilder(); + StringBuilder psqlCondition = new StringBuilder(); + mysqlCondition.append(cond); + psqlCondition.append(cond); + + if (fqnPrefix != null) { + String fqnPrefixHash = FullyQualifiedName.buildHash(fqnPrefix); + filter.queryParams.put("fqnPrefixHash", fqnPrefixHash); + filter.queryParams.put("concatFqnPrefixHash", fqnPrefixHash + ".%"); + String fqnCond = " AND (fqnHash LIKE :concatFqnPrefixHash OR fqnHash=:fqnPrefixHash)"; + mysqlCondition.append(fqnCond); + psqlCondition.append(fqnCond); + } + + if (entityType != null) { + mysqlCondition.append(" AND entityType=:entityType "); + psqlCondition.append(" AND entityType=:entityType "); + } + + return listCount( + getTableName(), + getNameHashColumn(), + filter.getQueryParams(), + mysqlCondition.toString(), + psqlCondition.toString()); + } + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT name, id, json FROM
AND " + + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " + + "ORDER BY name DESC,id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT name, id, json FROM
AND " + + "(name < :beforeName OR (name = :beforeName AND id < :beforeId)) " + + "ORDER BY name DESC,id DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY name,id", + connectionType = POSTGRES) + List listBefore( + @Define("table") String table, + @BindMap Map params, + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("beforeName") String beforeName, + @Bind("beforeId") String beforeId); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM
AND (
.name > :afterName OR (
.name = :afterName AND
.id > :afterId)) ORDER BY name,id LIMIT :limit", + connectionType = POSTGRES) + List listAfter( + @Define("table") String table, + @BindMap Map params, + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("afterName") String afterName, + @Bind("afterId") String afterId); + + @ConnectionAwareSqlQuery( + value = "SELECT json FROM doc_store WHERE name = :name AND entityType = 'EmailTemplate'", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT json FROM doc_store WHERE name = :name AND entityType = 'EmailTemplate'", + connectionType = POSTGRES) + String fetchEmailTemplateByName(@Bind("name") String name); + + @ConnectionAwareSqlQuery( + value = "SELECT json FROM doc_store WHERE entityType = 'EmailTemplate'", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT json FROM doc_store WHERE entityType = 'EmailTemplate'", + connectionType = POSTGRES) + List fetchAllEmailTemplates(); + + @ConnectionAwareSqlUpdate( + value = "DELETE FROM doc_store WHERE entityType = 'EmailTemplate'", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "DELETE FROM doc_store WHERE entityType = 'EmailTemplate'", + connectionType = POSTGRES) + void deleteEmailTemplates(); + } + + interface LearningResourceDAO extends EntityDAO { + @Override + default String getTableName() { + return "learning_resource_entity"; + } + + @Override + default Class getEntityClass() { + return LearningResource.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface ContextMemoryDAO extends EntityDAO { + @Override + default String getTableName() { + return "context_memory"; + } + + @Override + default Class getEntityClass() { + return ContextMemory.class; + } + + @Override + default String getNameHashColumn() { + return "nameHash"; + } + } + + interface SuggestionDAO { + default String getTableName() { + return "suggestions"; + } + + @ConnectionAwareSqlUpdate( + value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO suggestions(fqnHash, json) VALUES (:fqnHash, :json :: jsonb)", + connectionType = POSTGRES) + void insert(@BindFQN("fqnHash") String fullyQualifiedName, @Bind("json") String json); + + @ConnectionAwareSqlUpdate( + value = "UPDATE suggestions SET json = :json where id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE suggestions SET json = (:json :: jsonb) where id = :id", + connectionType = POSTGRES) + void update(@BindUUID("id") UUID id, @Bind("json") String json); + + @SqlQuery("SELECT json FROM suggestions WHERE id = :id") + String findById(@BindUUID("id") UUID id); + + @SqlUpdate("DELETE FROM suggestions WHERE id = :id") + void delete(@BindUUID("id") UUID id); + + @SqlUpdate("DELETE FROM suggestions WHERE fqnHash = :fqnHash") + void deleteByFQN(@BindUUID("fqnHash") String fullyQualifiedName); + + @ConnectionAwareSqlUpdate( + value = + "DELETE FROM suggestions suggestions WHERE JSON_EXTRACT(json, '$.createdBy.id') = :createdBy", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "DELETE FROM suggestions suggestions WHERE json #>> '{createdBy,id}' = :createdBy", + connectionType = POSTGRES) + void deleteByCreatedBy(@BindUUID("createdBy") UUID id); + + @SqlQuery("SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit") + List list( + @Bind("limit") int limit, + @Define("condition") String condition, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM suggestions ", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT count(*) FROM suggestions ", + connectionType = POSTGRES) + int listCount( + @Define("mysqlCond") String mysqlCond, + @Define("postgresCond") String postgresCond, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT updatedAt, json FROM suggestions " + + "ORDER BY updatedAt DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY updatedAt", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM (" + + "SELECT updatedAt, json FROM suggestions " + + "ORDER BY updatedAt DESC " + + "LIMIT :limit" + + ") last_rows_subquery ORDER BY updatedAt", + connectionType = POSTGRES) + List listBefore( + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("before") String before, + @BindMap Map params); + + @ConnectionAwareSqlQuery( + value = "SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = "SELECT json FROM suggestions ORDER BY updatedAt DESC LIMIT :limit", + connectionType = POSTGRES) + List listAfter( + @Define("mysqlCond") String mysqlCond, + @Define("psqlCond") String psqlCond, + @Bind("limit") int limit, + @Bind("after") String after, + @BindMap Map params); + } + + interface APICollectionDAO extends EntityDAO { + @Override + default String getTableName() { + return "api_collection_entity"; + } + + @Override + default Class getEntityClass() { + return APICollection.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface APIEndpointDAO extends EntityDAO { + @Override + default String getTableName() { + return "api_endpoint_entity"; + } + + @Override + default Class getEntityClass() { + return APIEndpoint.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface WorkflowDefinitionDAO extends EntityDAO { + @Override + default String getTableName() { + return "workflow_definition_entity"; + } + + @Override + default Class getEntityClass() { + return WorkflowDefinition.class; + } + + @Override + default String getNameHashColumn() { + return "fqnHash"; + } + } + + interface WorkflowInstanceTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "workflow_instance_time_series"; + } + } + + interface WorkflowInstanceStateTimeSeriesDAO extends EntityTimeSeriesDAO { + @Override + default String getTimeSeriesTableName() { + return "workflow_instance_state_time_series"; + } + + @SqlQuery( + value = + "SELECT json FROM workflow_instance_state_time_series " + + "WHERE workflowInstanceId = :workflowInstanceId AND stage = :stage ORDER BY timestamp DESC") + List listWorkflowInstanceStateForStage( + @Bind("workflowInstanceId") String workflowInstanceId, @Bind("stage") String stage); + + @SqlQuery( + value = + "SELECT json FROM workflow_instance_state_time_series " + + "WHERE workflowInstanceId = :workflowInstanceId ORDER BY timestamp ASC") + List listAllStatesForInstance(@Bind("workflowInstanceId") String workflowInstanceId); + } + + interface RecognizerFeedbackDAO { + @ConnectionAwareSqlUpdate( + value = "INSERT INTO recognizer_feedback_entity(json) VALUES (:json)", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "INSERT INTO recognizer_feedback_entity(json) VALUES (:json :: jsonb)", + connectionType = POSTGRES) + void insert(@Bind("json") String json); + + @SqlQuery("SELECT json FROM recognizer_feedback_entity WHERE id = :id") + String findById(@BindUUID("id") UUID id); + + @ConnectionAwareSqlUpdate( + value = "UPDATE recognizer_feedback_entity SET json = :json WHERE id = :id", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = "UPDATE recognizer_feedback_entity SET json = :json :: jsonb WHERE id = :id", + connectionType = POSTGRES) + void update(@BindUUID("id") UUID id, @Bind("json") String json); + + @SqlQuery("SELECT json FROM recognizer_feedback_entity WHERE entityLink = :entityLink") + List findByEntityLink(@Bind("entityLink") String entityLink); + + @SqlQuery("SELECT json FROM recognizer_feedback_entity WHERE tagFQN = :tagFQN") + List findByTagFQN(@Bind("tagFQN") String tagFQN); + + @SqlQuery("SELECT json FROM recognizer_feedback_entity WHERE status = :status") + List findByStatus(@Bind("status") String status); + + @SqlQuery("SELECT count(id) FROM recognizer_feedback_entity") + int count(); + + @SqlUpdate("DELETE FROM recognizer_feedback_entity WHERE id = :id") + void delete(@BindUUID("id") UUID id); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchReindexResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchReindexResource.java index 90de8338bd2e..b1d9d6112d24 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchReindexResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/search/SearchReindexResource.java @@ -31,7 +31,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexFailureDAO.SearchIndexFailureRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexFailureDAO.SearchIndexFailureRecord; import org.openmetadata.service.resources.Collection; import org.openmetadata.service.security.Authorizer; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java index ef22e7e8b395..30c910131bdc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java @@ -74,6 +74,7 @@ import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; import org.openmetadata.service.exception.SystemSettingsException; import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.jdbi3.EntityCacheInvalidator; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.GlossaryTermRepository; import org.openmetadata.service.jdbi3.ListFilter; @@ -1137,10 +1138,10 @@ public Response invalidateCacheForEntity( // 2. Guava L1 caches (CACHE_WITH_ID, CACHE_WITH_NAME) — the hot path on every entity // GET; without explicit eviction here, an admin force-invalidate wouldn't actually // take effect on the originating pod's in-memory cache. The static - // EntityRepository.invalidateCacheForEntity also propagates over the pub-sub channel + // EntityCacheInvalidator.invalidateCacheForEntity also propagates over the pub-sub channel // to other pods so multi-replica deploys all evict simultaneously. CacheBundle.invalidateEntity(type, id, normalizedFqn); - EntityRepository.invalidateCacheForEntity(type, id, normalizedFqn); + EntityCacheInvalidator.invalidateCacheForEntity(type, id, normalizedFqn); return Response.ok(Map.of("invalidated", true, "type", type)).build(); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexRetryWorker.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexRetryWorker.java index c242ba875f72..196e96c82f46 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexRetryWorker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexRetryWorker.java @@ -33,8 +33,8 @@ import org.openmetadata.service.apps.bundles.searchIndex.BulkSink; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexRetryQueueDAO.SearchIndexRetryRecord; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexRetryQueueDAO.SearchIndexRetryRecord; import org.openmetadata.service.workflows.searchIndex.ReindexingUtil; import os.org.opensearch.client.opensearch._types.OpenSearchException; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index d87c1c7b00c4..38c512485ea4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -266,7 +266,7 @@ public void deferIfFlushScopeActive( /** * Static variant of {@link #deferIfFlushScopeActive} for call sites that have no {@code * SearchRepository} instance in scope (e.g. the static {@code - * EntityRepository.invalidateCacheForTaggedEntities} ES-search loop). The {@code entityId}/{@code + * EntityCacheInvalidator.invalidateCacheForTaggedEntities} ES-search loop). The {@code entityId}/{@code * entityFqn} locator drives durable retry on a failed post-commit drain. */ public static void deferOrRunSearchWrite( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyConditionUpdater.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyConditionUpdater.java index 3eb7e2b625f9..3545609c70b7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyConditionUpdater.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/policyevaluator/PolicyConditionUpdater.java @@ -25,6 +25,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.utils.ResultList; import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.EntityCacheInvalidator; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.util.EntityUtil.Fields; @@ -140,7 +141,7 @@ public static void updateAllPolicyConditions(UnaryOperator conditionRewr // DAO.update skips EntityUpdater.invalidateCachesAfterStore, so the cached policy // still has the pre-rewrite condition embedded. Drop every cache variant for this // policy so the next read rebuilds from the freshly-updated row. - EntityRepository.invalidateCacheForEntity( + EntityCacheInvalidator.invalidateCacheForEntity( Entity.POLICY, policy.getId(), policy.getFullyQualifiedName()); anyChanged = true; LOG.info("Updated policy conditions for '{}'", policy.getFullyQualifiedName()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java index 902c646ce1e0..8837d62a72ea 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java @@ -20,7 +20,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; import org.openmetadata.service.security.session.SessionService; import org.openmetadata.service.security.session.SessionStatus; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java index 1dd664fbb4db..6916a7f932e0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/EntityUtil.java @@ -67,9 +67,9 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.exception.EntityNotFoundException; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityVersionPair; -import org.openmetadata.service.jdbi3.CollectionDAO.UsageDAO; +import org.openmetadata.service.jdbi3.AccessControlDAOs.UsageDAO; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipRecord; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityVersionPair; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; @@ -412,11 +412,11 @@ public static Map getLatestUsageForEntities( entityIds.stream().map(UUID::toString).collect(java.util.stream.Collectors.toList()); // Use the new batch query method for efficient bulk fetching - List + List usageDetailsList = usageDAO.getLatestUsageBatch(entityIdStrings); // Convert the list back to a map keyed by UUID - for (org.openmetadata.service.jdbi3.CollectionDAO.UsageDAO.UsageDetailsWithId usageWithId : + for (org.openmetadata.service.jdbi3.AccessControlDAOs.UsageDAO.UsageDetailsWithId usageWithId : usageDetailsList) { if (usageWithId != null && usageWithId.getEntityId() != null) { usageMap.put(UUID.fromString(usageWithId.getEntityId()), usageWithId.getUsageDetails()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/RdfBatchProcessorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/RdfBatchProcessorTest.java index bc555abe43a5..d08521d2d52f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/RdfBatchProcessorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/RdfBatchProcessorTest.java @@ -40,7 +40,7 @@ import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.type.Include; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipDAO; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipDAO; import org.openmetadata.service.rdf.RdfRepository; import org.openmetadata.service.rdf.storage.RdfStorageCircuitOpenException; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/RdfIndexAppTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/RdfIndexAppTest.java index b83bd4d3d80c..7b99e3954752 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/RdfIndexAppTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/RdfIndexAppTest.java @@ -34,8 +34,8 @@ import org.openmetadata.service.apps.bundles.rdf.distributed.RdfIndexJob; import org.openmetadata.service.apps.bundles.searchIndex.distributed.IndexJobStatus; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipObject; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipDAO; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipObject; import org.openmetadata.service.jdbi3.EntityDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.rdf.RdfRepository; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/distributed/DistributedRdfIndexCoordinatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/distributed/DistributedRdfIndexCoordinatorTest.java index 89567ad38988..c04c82a7c75c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/distributed/DistributedRdfIndexCoordinatorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/distributed/DistributedRdfIndexCoordinatorTest.java @@ -48,10 +48,10 @@ import org.openmetadata.service.apps.bundles.searchIndex.distributed.IndexJobStatus; import org.openmetadata.service.apps.bundles.searchIndex.distributed.ServerIdentityResolver; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexJobDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexJobDAO.RdfIndexJobRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexPartitionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.RdfIndexPartitionDAO.RdfAggregatedStatsRecord; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexJobDAO; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexJobDAO.RdfIndexJobRecord; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexPartitionDAO; +import org.openmetadata.service.jdbi3.RdfInfraDAOs.RdfIndexPartitionDAO.RdfAggregatedStatsRecord; @ExtendWith(MockitoExtension.class) class DistributedRdfIndexCoordinatorTest { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/IndexingFailureRecorderTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/IndexingFailureRecorderTest.java index 9bb7d0462a9e..b46119c2c053 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/IndexingFailureRecorderTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/IndexingFailureRecorderTest.java @@ -20,8 +20,8 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexFailureDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexFailureDAO.SearchIndexFailureRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexFailureDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexFailureDAO.SearchIndexFailureRecord; @ExtendWith(MockitoExtension.class) class IndexingFailureRecorderTest { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexAppTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexAppTest.java index 56858ce8bc54..171f6cabaf54 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexAppTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexAppTest.java @@ -43,8 +43,8 @@ import org.openmetadata.service.exception.AppException; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO.SearchIndexJobRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO.SearchIndexJobRecord; import org.openmetadata.service.search.SearchRepository; import org.quartz.JobExecutionContext; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexCoordinatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexCoordinatorTest.java index 5384cace87b0..765c00843019 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexCoordinatorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexCoordinatorTest.java @@ -54,13 +54,13 @@ import org.openmetadata.schema.system.EventPublisherJob; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO.SearchIndexJobRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.AggregatedStatsRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.EntityStatsRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.SearchIndexPartitionRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchReindexLockDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO.SearchIndexJobRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.AggregatedStatsRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.EntityStatsRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.SearchIndexPartitionRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchReindexLockDAO; @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutorTest.java index e3526f2da4fa..8025a1aa9bcf 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutorTest.java @@ -52,9 +52,9 @@ import org.openmetadata.service.apps.bundles.searchIndex.ReindexingMetrics; import org.openmetadata.service.apps.bundles.searchIndex.ReindexingProgressListener; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO; import org.openmetadata.service.search.DefaultRecreateHandler; import org.openmetadata.service.search.EntityReindexContext; import org.openmetadata.service.search.RecreateIndexHandler; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryManagerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryManagerTest.java index db28731de0ee..a1ba518046f4 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryManagerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryManagerTest.java @@ -43,12 +43,12 @@ import org.openmetadata.schema.system.EventPublisherJob; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO.SearchIndexJobRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.AggregatedStatsRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO.SearchIndexPartitionRecord; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchReindexLockDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO.SearchIndexJobRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.AggregatedStatsRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO.SearchIndexPartitionRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchReindexLockDAO; @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryOrphanDetectionTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryOrphanDetectionTest.java index 297116b2f0eb..ef03a4eaa42a 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryOrphanDetectionTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/JobRecoveryOrphanDetectionTest.java @@ -28,9 +28,9 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexJobDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexPartitionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchReindexLockDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexJobDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexPartitionDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchReindexLockDAO; @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.LENIENT) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/stats/StageStatsTrackerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/stats/StageStatsTrackerTest.java index 4f7c69e2b8d8..50637b6fee48 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/stats/StageStatsTrackerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/stats/StageStatsTrackerTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexServerStatsDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexServerStatsDAO; @ExtendWith(MockitoExtension.class) @DisplayName("StageStatsTracker Tests") diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/audit/AuditLogConsumerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/audit/AuditLogConsumerTest.java index e6bf4f00c3c3..ddafab3cb9c6 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/audit/AuditLogConsumerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/audit/AuditLogConsumerTest.java @@ -31,9 +31,9 @@ import org.openmetadata.schema.type.EventType; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.AccessControlDAOs.ChangeEventDAO; +import org.openmetadata.service.jdbi3.AccessControlDAOs.ChangeEventDAO.ChangeEventRecord; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.ChangeEventDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.ChangeEventDAO.ChangeEventRecord; import org.openmetadata.service.util.DIContainer; /** Unit tests for AuditLogConsumer batch processing and offset management. */ diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/CollectionDAOCompositionTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/CollectionDAOCompositionTest.java new file mode 100644 index 000000000000..616d69c333b7 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/CollectionDAOCompositionTest.java @@ -0,0 +1,96 @@ +/* + * Copyright 2024 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.jdbi3; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.jdbi.v3.sqlobject.CreateSqlObject; +import org.junit.jupiter.api.Test; + +/** + * Guards the CollectionDAO decomposition. CollectionDAO was split into domain aggregator + * interfaces (RdfInfraDAOs, SearchReindexDAOs, ...) that it `extends`. JDBI's SqlObjectFactory + * builds its handler map from {@code sqlObjectType.getMethods()}, which returns inherited public + * methods, so every {@code @CreateSqlObject} accessor moved to an extended interface must remain + * visible (and annotated) through CollectionDAO for the runtime wiring to keep working. + */ +class CollectionDAOCompositionTest { + + private static final Map MOVED_ACCESSOR_TO_INTERFACE = + Map.ofEntries( + Map.entry("rdfIndexJobDAO", "RdfInfraDAOs"), + Map.entry("rdfIndexServerStatsDAO", "RdfInfraDAOs"), + Map.entry("searchIndexJobDAO", "SearchReindexDAOs"), + Map.entry("searchIndexServerStatsDAO", "SearchReindexDAOs"), + Map.entry("aiApplicationDAO", "AiGovernanceDAOs"), + Map.entry("mcpServiceDAO", "AiGovernanceDAOs"), + Map.entry("feedDAO", "FeedDAOs"), + Map.entry("taskDAO", "FeedDAOs"), + Map.entry("tagUsageDAO", "ClassificationTagDAOs"), + Map.entry("classificationDAO", "ClassificationTagDAOs"), + Map.entry("testCaseDAO", "TimeSeriesDAOs"), + Map.entry("testCaseResultTimeSeriesDao", "TimeSeriesDAOs"), + Map.entry("activityStreamDAO", "ActivityAuditDAOs"), + Map.entry("auditLogDAO", "ActivityAuditDAOs"), + Map.entry("domainDAO", "GovernanceDAOs"), + Map.entry("dataContractDAO", "GovernanceDAOs"), + Map.entry("eventSubscriptionDAO", "EventSubscriptionDAOs"), + Map.entry("folderDAO", "KnowledgeAssetDAOs"), + Map.entry("knowledgePageDAO", "KnowledgeAssetDAOs"), + Map.entry("systemDAO", "SystemTokenDAOs"), + Map.entry("getTokenDAO", "SystemTokenDAOs"), + Map.entry("databaseDAO", "DataAssetServiceDAOs"), + Map.entry("dashboardDAO", "DataAssetServiceDAOs"), + Map.entry("containerDAO", "DataAssetServiceDAOs"), + Map.entry("dbServiceDAO", "DataAssetServiceDAOs"), + Map.entry("tableDAO", "EntityDataDAOs"), + Map.entry("pipelineDAO", "EntityDataDAOs"), + Map.entry("userDAO", "AccessControlDAOs"), + Map.entry("teamDAO", "AccessControlDAOs"), + Map.entry("changeEventDAO", "AccessControlDAOs"), + Map.entry("workflowDAO", "WorkflowDocStoreDAOs"), + Map.entry("oauthClientDAO", "OAuthDAOs"), + Map.entry("relationshipDAO", "CoreRelationshipDAOs"), + Map.entry("fieldRelationshipDAO", "CoreRelationshipDAOs"), + Map.entry("entityExtensionDAO", "CoreRelationshipDAOs")); + + @Test + void inheritedCreateSqlObjectAccessorsRemainVisibleAndAnnotated() throws NoSuchMethodException { + Set visibleMethods = + Arrays.stream(CollectionDAO.class.getMethods()) + .map(Method::getName) + .collect(Collectors.toSet()); + + for (Map.Entry entry : MOVED_ACCESSOR_TO_INTERFACE.entrySet()) { + String accessor = entry.getKey(); + assertTrue( + visibleMethods.contains(accessor), + accessor + " (moved to " + entry.getValue() + ") is not visible via getMethods()"); + Method method = CollectionDAO.class.getMethod(accessor); + assertNotNull( + method.getAnnotation(CreateSqlObject.class), + accessor + " lost its @CreateSqlObject annotation after the move"); + } + } + + @Test + void accessorsDeclaredDirectlyOnCollectionDaoStillWire() throws NoSuchMethodException { + assertNotNull(CollectionDAO.class.getMethod("assetDAO").getAnnotation(CreateSqlObject.class)); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryBulkFieldsTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryBulkFieldsTest.java index cb67ed376c7d..de81a6088a6c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryBulkFieldsTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryBulkFieldsTest.java @@ -36,7 +36,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipObject; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.EntityRelationshipObject; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.EntityUtil.RelationIncludes; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryCertificationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryCertificationTest.java index 966be3066a49..e0cad5c4c41d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryCertificationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryCertificationTest.java @@ -32,7 +32,7 @@ import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabelMetadata; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.CollectionDAO.TagUsageDAO; +import org.openmetadata.service.jdbi3.ClassificationTagDAOs.TagUsageDAO; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.EntityUtil.RelationIncludes; import org.openmetadata.service.util.FullyQualifiedName; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryEmbeddingsValidationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryEmbeddingsValidationTest.java index 6a4d216054b3..97fbee48c9bc 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryEmbeddingsValidationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryEmbeddingsValidationTest.java @@ -18,7 +18,7 @@ import org.openmetadata.schema.system.StepValidation; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; -import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO; +import org.openmetadata.service.jdbi3.SystemTokenDAOs.SystemDAO; import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.vector.VectorIndexService; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMissingIndexesTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMissingIndexesTest.java index c0caedd1d842..f269e9068a68 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMissingIndexesTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMissingIndexesTest.java @@ -15,7 +15,7 @@ import org.mockito.MockedStatic; import org.openmetadata.search.IndexMapping; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO; +import org.openmetadata.service.jdbi3.SystemTokenDAOs.SystemDAO; import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.search.SearchRepository; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/CachedPermissionEvaluationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/CachedPermissionEvaluationTest.java index a47ab6434352..fc16610af214 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/CachedPermissionEvaluationTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/CachedPermissionEvaluationTest.java @@ -39,7 +39,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EntityCaches; import org.openmetadata.service.jdbi3.PolicyRepository; import org.openmetadata.service.jdbi3.RoleRepository; import org.openmetadata.service.jdbi3.TeamRepository; @@ -72,7 +72,7 @@ private static void setupMocks() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_NAME.get( + EntityCaches.CACHE_WITH_NAME.get( new ImmutablePair<>(Entity.USER, i.getArgument(1))), User.class)); @@ -84,7 +84,7 @@ private static void setupMocks() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.TEAM, i.getArgument(1))), Team.class)); @@ -96,7 +96,7 @@ private static void setupMocks() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.ROLE, i.getArgument(1))), Role.class)); @@ -108,7 +108,7 @@ private static void setupMocks() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.POLICY, i.getArgument(1))), Policy.class)); } @@ -139,7 +139,7 @@ private static void setupTeamHierarchy() { .withName("testUser") .withRoles(toEntityReferences(userRoles)) .withTeams(List.of(teamA.getEntityReference(), teamB.getEntityReference())); - EntityRepository.CACHE_WITH_NAME.put( + EntityCaches.CACHE_WITH_NAME.put( new ImmutablePair<>(Entity.USER, "testUser"), JsonUtils.pojoToJson(testUser)); } @@ -277,7 +277,7 @@ private static List createRoles(String prefix) { String name = prefix + "_role_" + i; List policies = toEntityReferences(createPolicies(name)); Role role = new Role().withName(name).withId(UUID.randomUUID()).withPolicies(policies); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.ROLE, role.getId()), JsonUtils.pojoToJson(role)); roles.add(role); } @@ -291,7 +291,7 @@ private static List createPolicies(String prefix) { Policy policy = new Policy().withName(name).withId(UUID.randomUUID()).withRules(createRules(name)); policies.add(policy); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.POLICY, policy.getId()), JsonUtils.pojoToJson(policy)); } return policies; @@ -324,7 +324,7 @@ private static Team createTeam( .withDefaultRoles(toEntityReferences(roles)) .withPolicies(toEntityReferences(policies)) .withParents(parentList); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, team.getId()), JsonUtils.pojoToJson(team)); return team; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java index 1113ba5a26fe..ca38341d6335 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/RuleEvaluatorTest.java @@ -49,7 +49,7 @@ import org.openmetadata.service.jdbi3.DatabaseRepository; import org.openmetadata.service.jdbi3.DatabaseSchemaRepository; import org.openmetadata.service.jdbi3.DomainRepository; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EntityCaches; import org.openmetadata.service.jdbi3.GlossaryRepository; import org.openmetadata.service.jdbi3.TableRepository; import org.openmetadata.service.jdbi3.TeamRepository; @@ -87,14 +87,14 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.TEAM, i.getArgument(0))), Team.class)); Mockito.when(teamRepository.getReference(any(UUID.class), any(Include.class))) .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.TEAM, i.getArgument(0))), Team.class) .getEntityReference()); @@ -103,7 +103,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_NAME.get( + EntityCaches.CACHE_WITH_NAME.get( new ImmutablePair<>(Entity.TEAM, i.getArgument(0))), Team.class)); @@ -113,7 +113,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.TEAM, i.getArgument(1))), Team.class)); @@ -123,7 +123,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.TEAM, i.getArgument(1))), Team.class)); @@ -152,7 +152,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.DOMAIN, i.getArgument(1))), Domain.class)); @@ -178,9 +178,9 @@ public static void setup() { DatabaseSchema schema = new DatabaseSchema().withId(UUID.randomUUID()).withName("testSchema"); schema.setDatabase(databaseRef); database.setOwners(List.of(ownerRef)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DATABASE_SCHEMA, schema.getId()), JsonUtils.pojoToJson(schema)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DATABASE, database.getId()), JsonUtils.pojoToJson(database)); Mockito.when(databaseSchemaRepository.getParentEntity(any(DatabaseSchema.class), anyString())) .thenAnswer( @@ -189,7 +189,7 @@ public static void setup() { EntityReference dbRef = cachedSchema.getDatabase(); if (dbRef == null) return null; return JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.DATABASE, dbRef.getId())), Database.class); }); @@ -208,9 +208,9 @@ public static void setup() { .withName("testDataProduct") .withFullyQualifiedName("testDataProduct") .withDomains(List.of(domain.getEntityReference())); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, domain.getId()), JsonUtils.pojoToJson(domain)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DATA_PRODUCT, dataProduct.getId()), JsonUtils.pojoToJson(dataProduct)); resourceContextDataProduct = @@ -360,7 +360,7 @@ void test_isReviewer() { .withName("testGlossary") .withReviewers(List.of(reviewerRef)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.GLOSSARY, glossary.getId()), JsonUtils.pojoToJson(glossary)); SubjectContext subjectContext = new SubjectContext(reviewer, null); @@ -730,7 +730,7 @@ private Team createTeam(String teamName, String parentName) { Entity.getEntityReferenceById(Entity.TEAM, parentId, Include.NON_DELETED); team.setParents(listOf(parentTeam)); } - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, team.getId()), JsonUtils.pojoToJson(team)); return team; } @@ -746,7 +746,7 @@ private Team createTeamWithRole(String teamName, String parentName) { team.getInheritedRoles().addAll(listOrEmpty(parentTeam.getDefaultRoles())); team.getInheritedRoles().addAll(listOrEmpty(parentTeam.getInheritedRoles())); } - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, team.getId()), JsonUtils.pojoToJson(team)); return team; } @@ -754,7 +754,7 @@ private Team createTeamWithRole(String teamName, String parentName) { private Role createRole(String roleName) { UUID roleId = UUID.nameUUIDFromBytes(roleName.getBytes(StandardCharsets.UTF_8)); Role role = new Role().withName(roleName).withId(roleId); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.ROLE, role.getId()), JsonUtils.pojoToJson(role)); return role; } @@ -795,14 +795,14 @@ void test_hasDomain() { .withFullyQualifiedName("Marketing"); // Cache domains for Entity.getEntity calls - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, rootDomain.getId()), JsonUtils.pojoToJson(rootDomain)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, subDomain.getId()), JsonUtils.pojoToJson(subDomain)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, subSubDomain.getId()), JsonUtils.pojoToJson(subSubDomain)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, unrelatedDomain.getId()), JsonUtils.pojoToJson(unrelatedDomain)); @@ -885,14 +885,14 @@ void test_hasDomain_withComplexHierarchy() { .withParent(company.getEntityReference()); // Cache domains - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, company.getId()), JsonUtils.pojoToJson(company)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, engineering.getId()), JsonUtils.pojoToJson(engineering)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, dataEngineering.getId()), JsonUtils.pojoToJson(dataEngineering)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, analytics.getId()), JsonUtils.pojoToJson(analytics)); // Test: User with Engineering domain should have access to DataEngineering resources @@ -994,7 +994,7 @@ void test_hasDomain_edgeCases_realFQNFormat() { private Domain createDomain(String name, String fqn) { Domain domain = new Domain().withId(UUID.randomUUID()).withName(name).withFullyQualifiedName(fqn); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, domain.getId()), JsonUtils.pojoToJson(domain)); return domain; } @@ -1026,13 +1026,13 @@ void test_hasDomain_hierarchicalAccess_parentDomainUsersCanAccessSubDomainResour .withParent(accountingDomain.getEntityReference()); // Cache domains for Entity.getEntity calls - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, financeDomain.getId()), JsonUtils.pojoToJson(financeDomain)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, accountingDomain.getId()), JsonUtils.pojoToJson(accountingDomain)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.DOMAIN, payrollDomain.getId()), JsonUtils.pojoToJson(payrollDomain)); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectCacheTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectCacheTest.java index e45b3fb62ece..4dc977801ef7 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectCacheTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectCacheTest.java @@ -41,7 +41,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EntityCaches; import org.openmetadata.service.jdbi3.PolicyRepository; import org.openmetadata.service.jdbi3.RoleRepository; import org.openmetadata.service.jdbi3.TeamRepository; @@ -68,7 +68,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_NAME.get( + EntityCaches.CACHE_WITH_NAME.get( new ImmutablePair<>(Entity.USER, i.getArgument(1))), User.class)); @@ -80,7 +80,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.TEAM, i.getArgument(1))), Team.class)); @@ -92,7 +92,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.ROLE, i.getArgument(1))), Role.class)); @@ -104,7 +104,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.POLICY, i.getArgument(1))), Policy.class)); @@ -124,7 +124,7 @@ public static void setup() { .withName("testUser") .withRoles(userRolesRef) .withTeams(List.of(team11.getEntityReference())); - EntityRepository.CACHE_WITH_NAME.put( + EntityCaches.CACHE_WITH_NAME.put( new ImmutablePair<>(Entity.USER, "testUser"), JsonUtils.pojoToJson(user)); } @@ -287,7 +287,7 @@ void testBotUserDoesNotInheritTeamPolicies() { .withRoles(toEntityReferences(botRoles)) .withTeams(List.of(team11.getEntityReference())) .withIsBot(true); - EntityRepository.CACHE_WITH_NAME.put( + EntityCaches.CACHE_WITH_NAME.put( new ImmutablePair<>(Entity.USER, "botUser"), JsonUtils.pojoToJson(botUser)); List botPolicies = SubjectCache.getPolicies("botUser"); @@ -307,7 +307,7 @@ private static List getRoles(String prefix) { String name = prefix + "_role_" + i; List policies = toEntityReferences(getPolicies(name)); Role role = new Role().withName(name).withId(UUID.randomUUID()).withPolicies(policies); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.ROLE, role.getId()), JsonUtils.pojoToJson(role)); roles.add(role); } @@ -321,7 +321,7 @@ private static List getPolicies(String prefix) { Policy policy = new Policy().withName(name).withId(UUID.randomUUID()).withRules(getRules(name)); policies.add(policy); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.POLICY, policy.getId()), JsonUtils.pojoToJson(policy)); } return policies; @@ -354,7 +354,7 @@ private static Team createTeam( .withDefaultRoles(toEntityReferences(roles)) .withPolicies(toEntityReferences(policies)) .withParents(parentList); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, team.getId()), JsonUtils.pojoToJson(team)); return team; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectContextTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectContextTest.java index 5c0c36255233..50375b1400a2 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectContextTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/SubjectContextTest.java @@ -41,7 +41,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EntityCaches; import org.openmetadata.service.jdbi3.PolicyRepository; import org.openmetadata.service.jdbi3.RoleRepository; import org.openmetadata.service.jdbi3.TeamRepository; @@ -81,7 +81,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_NAME.get( + EntityCaches.CACHE_WITH_NAME.get( new ImmutablePair<>(Entity.USER, i.getArgument(1))), User.class)); @@ -93,7 +93,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.TEAM, i.getArgument(1))), Team.class)); @@ -105,7 +105,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.ROLE, i.getArgument(1))), Role.class)); @@ -117,7 +117,7 @@ public static void setup() { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(Entity.POLICY, i.getArgument(1))), Policy.class)); @@ -162,7 +162,7 @@ public static void setup() { .withName("user") .withRoles(userRolesRef) .withTeams(List.of(team111.getEntityReference())); - EntityRepository.CACHE_WITH_NAME.put( + EntityCaches.CACHE_WITH_NAME.put( new ImmutablePair<>(Entity.USER, "user"), JsonUtils.pojoToJson(user)); } @@ -254,7 +254,7 @@ private static List getRoles(String prefix) { String name = prefix + "_role_" + i; List policies = toEntityReferences(getPolicies(name)); Role role = new Role().withName(name).withId(UUID.randomUUID()).withPolicies(policies); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.ROLE, role.getId()), JsonUtils.pojoToJson(role)); roles.add(role); } @@ -268,7 +268,7 @@ private static List getPolicies(String prefix) { Policy policy = new Policy().withName(name).withId(UUID.randomUUID()).withRules(getRules(name)); policies.add(policy); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.POLICY, policy.getId()), JsonUtils.pojoToJson(policy)); } return policies; @@ -326,7 +326,7 @@ private static Team createTeam( .withDefaultRoles(toEntityReferences(roles)) .withPolicies(toEntityReferences(policies)) .withParents(parentList); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, team.getId()), JsonUtils.pojoToJson(team)); return team; } @@ -382,12 +382,12 @@ void testCircularDependencyInTeamHierarchy() { .withId(UUID.randomUUID()) .withDefaultRoles(toEntityReferences(circularTeamRoles)) .withPolicies(toEntityReferences(circularTeamPolicies)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, circularTeam.getId()), JsonUtils.pojoToJson(circularTeam)); // Create circular reference - team points to itself as parent circularTeam.setParents(List.of(circularTeam.getEntityReference())); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, circularTeam.getId()), JsonUtils.pojoToJson(circularTeam)); // Test getRolesForTeams - should not cause StackOverflowError @@ -408,7 +408,7 @@ void testCircularDependencyInTeamHierarchy() { .withId(UUID.randomUUID()) .withDefaultRoles(toEntityReferences(teamARoles)) .withPolicies(toEntityReferences(teamAPolicies)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, teamA.getId()), JsonUtils.pojoToJson(teamA)); List teamBRoles = getRoles("teamB"); @@ -419,15 +419,15 @@ void testCircularDependencyInTeamHierarchy() { .withId(UUID.randomUUID()) .withDefaultRoles(toEntityReferences(teamBRoles)) .withPolicies(toEntityReferences(teamBPolicies)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, teamB.getId()), JsonUtils.pojoToJson(teamB)); // Create circular dependency: teamA -> teamB -> teamA teamA.setParents(List.of(teamB.getEntityReference())); teamB.setParents(List.of(teamA.getEntityReference())); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, teamA.getId()), JsonUtils.pojoToJson(teamA)); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(Entity.TEAM, teamB.getId()), JsonUtils.pojoToJson(teamB)); // Test getRolesForTeams - should not cause StackOverflowError @@ -445,7 +445,7 @@ void testCircularDependencyInTeamHierarchy() { .withName("circularUser") .withRoles(new ArrayList<>()) .withTeams(List.of(teamA.getEntityReference())); - EntityRepository.CACHE_WITH_NAME.put( + EntityCaches.CACHE_WITH_NAME.put( new ImmutablePair<>(Entity.USER, "circularUser"), JsonUtils.pojoToJson(userWithCircularTeam)); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/TaskResourceContextTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/TaskResourceContextTest.java index 1492136e9aa8..58c63841cf68 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/TaskResourceContextTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/security/policyevaluator/TaskResourceContextTest.java @@ -31,7 +31,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EntityCaches; import org.openmetadata.service.jdbi3.TableRepository; class TaskResourceContextTest { @@ -71,7 +71,7 @@ static void setup() throws Exception { .withName(target.getName()) .withFullyQualifiedName(target.getFullyQualifiedName()); - EntityRepository.CACHE_WITH_ID.put( + EntityCaches.CACHE_WITH_ID.put( new ImmutablePair<>(TARGET_ENTITY_TYPE, target.getId()), JsonUtils.pojoToJson(target)); // Repository.getOwners(reference) → returns the entity's owners @@ -81,14 +81,14 @@ static void setup() throws Exception { .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_ID.get( + EntityCaches.CACHE_WITH_ID.get( new ImmutablePair<>(TARGET_ENTITY_TYPE, i.getArgument(0))), Table.class)); Mockito.when(targetRepository.findByName(anyString(), any())) .thenAnswer( i -> JsonUtils.readValue( - EntityRepository.CACHE_WITH_NAME.get( + EntityCaches.CACHE_WITH_NAME.get( new ImmutablePair<>(TARGET_ENTITY_TYPE, i.getArgument(0))), Table.class)); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityUtilTest.java index d514c09946da..75264c5ef15d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityUtilTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/EntityUtilTest.java @@ -43,9 +43,9 @@ import org.openmetadata.schema.type.UsageStats; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.AccessControlDAOs.UsageDAO; +import org.openmetadata.service.jdbi3.AccessControlDAOs.UsageDAO.UsageDetailsWithId; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.UsageDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.UsageDAO.UsageDetailsWithId; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.resources.feeds.MessageParser; From ee8966c468b0fdb5249b8567ca68fef56da071f9 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 16:19:25 -0700 Subject: [PATCH 02/13] perf(jdbi3): index data_contract lookups (A3) + keyset tag_usage cleanup (E1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A3 — getContractByEntityId full-scanned data_contract_entity filtering nested $.entity.id / $.entity.type via JSON_EXTRACT on the per-entity contract enforcement write path. Add VIRTUAL (MySQL) / STORED (Postgres) generated columns contractEntityId / contractEntityType + a composite index, and rewrite the query to seek on those columns plus the existing indexed `deleted` column. Validated on MySQL 8.0 + Postgres 16: generated columns populate, query returns only non-deleted matches, EXPLAIN shows an index seek (ref / Index Scan), and the guarded migration is idempotent. E1 — TagUsageCleanup.performCleanup walked the full tag_usage table with offset += batchSize (O(n^2) deep-offset). Switch to keyset pagination on the unique (source, tagFQNHash, targetFQNHash) key. Per-engine SQL: MySQL needs the expanded-OR form (its optimizer makes it an index range seek; a row-constructor comparison scans the whole index from the start — verified Handler_read_next), Postgres seeks optimally with the row-constructor (Index Only Scan). Validated on both engines: a batchSize=7 drive over 300 rows covers all 300 exactly once with zero duplicates. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../native/2.0.0/mysql/schemaChanges.sql | 53 +++++++++++++++++ .../native/2.0.0/postgres/schemaChanges.sql | 12 ++++ .../service/jdbi3/ClassificationTagDAOs.java | 29 +++++++-- .../service/jdbi3/GovernanceDAOs.java | 4 +- .../service/util/TagUsageCleanup.java | 59 +++++++++++-------- 5 files changed, 128 insertions(+), 29 deletions(-) diff --git a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql index 77ca00635852..bb85de91f52d 100644 --- a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql @@ -400,3 +400,56 @@ SET @ddl = ( PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +-- Perf (A3): getContractByEntityId filtered $.entity.id / $.entity.type via JSON_EXTRACT +-- (nested, unindexed paths) on the per-entity contract-enforcement write path, full-scanning +-- data_contract_entity. Add VIRTUAL generated columns mirroring those paths plus a composite +-- index so the lookup becomes an index seek. VIRTUAL columns need no table rewrite. MySQL has +-- no ADD COLUMN/KEY IF NOT EXISTS, so guard each add via information_schema. +SET @ddl = ( + SELECT IF( + EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = 'data_contract_entity' + AND column_name = 'contractEntityId' + ), + 'SELECT 1', + 'ALTER TABLE data_contract_entity ADD COLUMN contractEntityId VARCHAR(36) GENERATED ALWAYS AS (json ->> ''$.entity.id'') VIRTUAL' + ) +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @ddl = ( + SELECT IF( + EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name = 'data_contract_entity' + AND column_name = 'contractEntityType' + ), + 'SELECT 1', + 'ALTER TABLE data_contract_entity ADD COLUMN contractEntityType VARCHAR(256) GENERATED ALWAYS AS (json ->> ''$.entity.type'') VIRTUAL' + ) +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @ddl = ( + SELECT IF( + EXISTS ( + SELECT 1 FROM information_schema.statistics + WHERE table_schema = DATABASE() + AND table_name = 'data_contract_entity' + AND index_name = 'idx_data_contract_entity_contract_entity' + ), + 'SELECT 1', + 'ALTER TABLE data_contract_entity ADD KEY idx_data_contract_entity_contract_entity (contractEntityId, contractEntityType)' + ) +); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql index 7585c90eadde..33ba39a50f78 100644 --- a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql @@ -350,3 +350,15 @@ ALTER TABLE user_entity DROP COLUMN IF EXISTS isBot; ALTER TABLE user_entity ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS ((json ->> 'isBot')::boolean) STORED NOT NULL; CREATE INDEX IF NOT EXISTS idx_isBot ON user_entity (isBot); + +-- Perf (A3): getContractByEntityId filtered json#>>'{entity,id}' / '{entity,type}' (nested, +-- unindexed paths) on the per-entity contract-enforcement write path, full-scanning +-- data_contract_entity. Add STORED generated columns mirroring those paths plus a composite +-- index so the lookup becomes an index seek. The #>> operator on jsonb is immutable, so it is +-- valid in a STORED generated column; STORED backfills existing rows automatically. +ALTER TABLE data_contract_entity + ADD COLUMN IF NOT EXISTS contractEntityId VARCHAR(36) GENERATED ALWAYS AS (json #>> '{entity,id}') STORED; +ALTER TABLE data_contract_entity + ADD COLUMN IF NOT EXISTS contractEntityType VARCHAR(256) GENERATED ALWAYS AS (json #>> '{entity,type}') STORED; +CREATE INDEX IF NOT EXISTS idx_data_contract_entity_contract_entity + ON data_contract_entity (contractEntityId, contractEntityType); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java index 542d1e6a9de9..ca4434767eba 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java @@ -1153,11 +1153,32 @@ void deleteTagsBatchInternal( @SqlQuery("SELECT COUNT(*) FROM tag_usage") long getTotalTagUsageCount(); - @SqlQuery( - "SELECT source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedAt, appliedBy, metadata FROM tag_usage ORDER BY source, tagFQNHash LIMIT :limit OFFSET :offset") + // Keyset pagination over the unique (source, tagFQNHash, targetFQNHash) key. The expanded + // OR form is required on MySQL — its optimizer turns it into an index range seek, whereas a + // row-constructor comparison scans the whole index from the start (verified via + // Handler_read_next). Postgres seeks optimally with the row-constructor (Index Only Scan). + @ConnectionAwareSqlQuery( + value = + "SELECT source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedAt, appliedBy, metadata " + + "FROM tag_usage " + + "WHERE source > :lastSource " + + "OR (source = :lastSource AND tagFQNHash > :lastTagFQNHash) " + + "OR (source = :lastSource AND tagFQNHash = :lastTagFQNHash AND targetFQNHash > :lastTargetFQNHash) " + + "ORDER BY source, tagFQNHash, targetFQNHash LIMIT :limit", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedAt, appliedBy, metadata " + + "FROM tag_usage " + + "WHERE (source, tagFQNHash, targetFQNHash) > (:lastSource, :lastTagFQNHash, :lastTargetFQNHash) " + + "ORDER BY source, tagFQNHash, targetFQNHash LIMIT :limit", + connectionType = POSTGRES) @RegisterRowMapper(TagUsageObjectMapper.class) - List getAllTagUsagesPaginated( - @Bind("offset") long offset, @Bind("limit") int limit); + List getTagUsagesAfter( + @Bind("lastSource") int lastSource, + @Bind("lastTagFQNHash") String lastTagFQNHash, + @Bind("lastTargetFQNHash") String lastTargetFQNHash, + @Bind("limit") int limit); @SqlUpdate( "DELETE FROM tag_usage WHERE source = :source AND tagFQNHash = :tagFQNHash AND targetFQNHash = :targetFQNHash") diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java index c61bb2891d1b..da64cee8e48b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java @@ -202,11 +202,11 @@ default String getNameHashColumn() { @ConnectionAwareSqlQuery( value = - "SELECT json FROM data_contract_entity WHERE JSON_EXTRACT(json, '$.entity.id') = :entityId AND JSON_EXTRACT(json, '$.entity.type') = :entityType AND (JSON_EXTRACT(json, '$.deleted') IS NULL OR JSON_EXTRACT(json, '$.deleted') = false) LIMIT 1", + "SELECT json FROM data_contract_entity WHERE contractEntityId = :entityId AND contractEntityType = :entityType AND deleted = FALSE LIMIT 1", connectionType = MYSQL) @ConnectionAwareSqlQuery( value = - "SELECT json FROM data_contract_entity WHERE json#>>'{entity,id}' = :entityId AND json#>>'{entity,type}' = :entityType AND (json->>'deleted' IS NULL OR json->>'deleted' = 'false') LIMIT 1", + "SELECT json FROM data_contract_entity WHERE contractEntityId = :entityId AND contractEntityType = :entityType AND deleted = FALSE LIMIT 1", connectionType = POSTGRES) String getContractByEntityId( @Bind("entityId") String entityId, @Bind("entityType") String entityType); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/TagUsageCleanup.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/TagUsageCleanup.java index d629670e614c..5bc8a9272999 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/TagUsageCleanup.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/TagUsageCleanup.java @@ -88,39 +88,52 @@ public TagCleanupResult performCleanup(int batchSize) { totalTagUsages, batchSize); - long offset = 0; + int lastSource = -1; + String lastTagFQNHash = ""; + String lastTargetFQNHash = ""; int processedCount = 0; int batchNumber = 1; + boolean hasMore = true; - while (offset < totalTagUsages) { - LOG.info("Processing batch {} (offset: {}, limit: {})", batchNumber, offset, batchSize); + while (hasMore) { + LOG.info( + "Processing batch {} (after source={}, tagFQNHash={})", + batchNumber, + lastSource, + lastTagFQNHash); List tagUsageBatch = - collectionDAO.tagUsageDAO().getAllTagUsagesPaginated(offset, batchSize); + collectionDAO + .tagUsageDAO() + .getTagUsagesAfter(lastSource, lastTagFQNHash, lastTargetFQNHash, batchSize); if (tagUsageBatch.isEmpty()) { LOG.info("No more tag usages to process"); - break; - } - - for (CollectionDAO.TagUsageObject tagUsage : tagUsageBatch) { - OrphanedTagUsage orphan = validateTagUsage(tagUsage); - if (orphan != null) { - result.getOrphanedTagUsages().add(orphan); - result.getOrphansBySource().merge(orphan.getSourceName(), 1, Integer::sum); + hasMore = false; + } else { + for (CollectionDAO.TagUsageObject tagUsage : tagUsageBatch) { + OrphanedTagUsage orphan = validateTagUsage(tagUsage); + if (orphan != null) { + result.getOrphanedTagUsages().add(orphan); + result.getOrphansBySource().merge(orphan.getSourceName(), 1, Integer::sum); + } + processedCount++; } - processedCount++; - } - - offset += tagUsageBatch.size(); - batchNumber++; - if (processedCount % (batchSize * 10) == 0 || offset >= totalTagUsages) { - LOG.info( - "Progress: {}/{} tag usages processed, {} orphaned tag usages found", - processedCount, - totalTagUsages, - result.getOrphanedTagUsages().size()); + CollectionDAO.TagUsageObject lastRow = tagUsageBatch.getLast(); + lastSource = lastRow.getSource(); + lastTagFQNHash = lastRow.getTagFQNHash(); + lastTargetFQNHash = lastRow.getTargetFQNHash(); + batchNumber++; + hasMore = tagUsageBatch.size() == batchSize; + + if (processedCount % (batchSize * 10) == 0 || !hasMore) { + LOG.info( + "Progress: {}/{} tag usages processed, {} orphaned tag usages found", + processedCount, + totalTagUsages, + result.getOrphanedTagUsages().size()); + } } } From bc157cf71f1e536e00fb21d4f1a0fd0517e52b59 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 16:30:30 -0700 Subject: [PATCH 03/13] perf(jdbi3): batch field_relationship inserts for mention write paths (C1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FieldRelationshipDAO was the one high-volume relationship DAO with no batch insert, so FeedRepository.storeMentions and TaskRepository.storeMentions fired one round trip per distinct mention. Add insertMany(List) mirroring EntityRelationshipDAO.bulkInsertTo: a @BindBeanList multi-row VALUES with INSERT IGNORE (MySQL) / ON CONFLICT (fromFQNHash, toFQNHash, relation) DO NOTHING (Postgres), omitting the nullable json column so no per-row ::jsonb cast is needed. Both mention callers now build the bean list (pre-hashing FQNs via FullyQualifiedName.buildHash — the same hash @BindFQN applies) and issue one insertMany; empty lists are guarded so the VALUES clause is never empty. Pipeline task-owner inserts are intentionally left on the single-row path: they pass buildHash(fqn) into a @BindFQN param (a separate double-hash concern), so a bean-bound batch would change the stored hash — out of scope here. Validated on MySQL 8.0 + Postgres 16: a multi-row batch inserts all distinct rows, a follow-up batch with duplicate keys is ignored with no error, and json stays NULL — matching the single-row insert semantics. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/CoreRelationshipDAOs.java | 28 +++++++++++++ .../service/jdbi3/FeedRepository.java | 40 ++++++++++++------- .../service/jdbi3/TaskRepository.java | 40 ++++++++++++------- 3 files changed, 78 insertions(+), 30 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CoreRelationshipDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CoreRelationshipDAOs.java index 4cb2918cde74..ec12f253c29b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CoreRelationshipDAOs.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CoreRelationshipDAOs.java @@ -1432,6 +1432,34 @@ void insert( @Bind("relation") int relation, @Bind("json") String json); + // Batch counterpart of insert(...) for INSERT-only callers that loop one round trip per row. + // Binds the FieldRelationship bean's pre-hashed fromFQNHash/toFQNHash directly (callers must + // hash via FullyQualifiedName.buildHash, mirroring @BindFQN on the single-row insert), and + // omits the nullable json column so the per-row Postgres ::jsonb cast is unnecessary. + @ConnectionAwareSqlUpdate( + value = + "INSERT IGNORE INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation) " + + "VALUES ", + connectionType = MYSQL) + @ConnectionAwareSqlUpdate( + value = + "INSERT INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation) " + + "VALUES ON CONFLICT (fromFQNHash, toFQNHash, relation) DO NOTHING", + connectionType = POSTGRES) + void insertMany( + @BindBeanList( + value = "values", + propertyNames = { + "fromFQNHash", + "toFQNHash", + "fromFQN", + "toFQN", + "fromType", + "toType", + "relation" + }) + List values); + @ConnectionAwareSqlUpdate( value = "INSERT INTO field_relationship(fromFQNHash, toFQNHash, fromFQN, toFQN, fromType, toType, relation, jsonSchema, json) " diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java index 124eaca672b7..844c38941267 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java @@ -98,6 +98,7 @@ import org.openmetadata.service.formatter.decorators.MessageDecorator; import org.openmetadata.service.formatter.util.FeedMessage; import org.openmetadata.service.governance.workflows.WorkflowHandler; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.FieldRelationshipDAO.FieldRelationship; import org.openmetadata.service.resources.databases.DatasourceConfig; import org.openmetadata.service.resources.feeds.FeedResource; import org.openmetadata.service.resources.feeds.FeedUtil; @@ -670,21 +671,30 @@ private void storeMentions(Thread thread, String message) { // Create relationship for users, teams, and other entities that are mentioned in the post // Multiple mentions of the same entity is handled by taking distinct mentions List mentions = MessageParser.getEntityLinks(message); - - mentions.stream() - .distinct() - .forEach( - mention -> - dao.fieldRelationshipDAO() - .insert( - mention.getFullyQualifiedFieldValue(), - thread.getId().toString(), - mention.getFullyQualifiedFieldValue(), - thread.getId().toString(), - mention.getFullyQualifiedFieldType(), - Entity.THREAD, - Relationship.MENTIONED_IN.ordinal(), - null)); + String threadId = thread.getId().toString(); + String threadIdHash = FullyQualifiedName.buildHash(threadId); + + List relationships = + mentions.stream() + .distinct() + .map( + mention -> { + FieldRelationship relationship = new FieldRelationship(); + relationship.setFromFQNHash( + FullyQualifiedName.buildHash(mention.getFullyQualifiedFieldValue())); + relationship.setToFQNHash(threadIdHash); + relationship.setFromFQN(mention.getFullyQualifiedFieldValue()); + relationship.setToFQN(threadId); + relationship.setFromType(mention.getFullyQualifiedFieldType()); + relationship.setToType(Entity.THREAD); + relationship.setRelation(Relationship.MENTIONED_IN.ordinal()); + return relationship; + }) + .toList(); + + if (!relationships.isEmpty()) { + dao.fieldRelationshipDAO().insertMany(relationships); + } } @Transaction diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TaskRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TaskRepository.java index b9c04211e32d..d1b91fd9f8ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TaskRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TaskRepository.java @@ -58,6 +58,7 @@ import org.openmetadata.service.events.lifecycle.handlers.IncidentTcrsSyncHandler; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.governance.workflows.WorkflowHandler; +import org.openmetadata.service.jdbi3.CoreRelationshipDAOs.FieldRelationshipDAO.FieldRelationship; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.security.AuthRequest; @@ -597,21 +598,30 @@ private void storeMentions(Task task, String message) { } List mentions = MessageParser.getEntityLinks(message); - mentions.stream() - .distinct() - .forEach( - mention -> - daoCollection - .fieldRelationshipDAO() - .insert( - mention.getFullyQualifiedFieldValue(), - task.getId().toString(), - mention.getFullyQualifiedFieldValue(), - task.getId().toString(), - mention.getFullyQualifiedFieldType(), - Entity.TASK, - Relationship.MENTIONED_IN.ordinal(), - null)); + String taskId = task.getId().toString(); + String taskIdHash = FullyQualifiedName.buildHash(taskId); + + List relationships = + mentions.stream() + .distinct() + .map( + mention -> { + FieldRelationship relationship = new FieldRelationship(); + relationship.setFromFQNHash( + FullyQualifiedName.buildHash(mention.getFullyQualifiedFieldValue())); + relationship.setToFQNHash(taskIdHash); + relationship.setFromFQN(mention.getFullyQualifiedFieldValue()); + relationship.setToFQN(taskId); + relationship.setFromType(mention.getFullyQualifiedFieldType()); + relationship.setToType(Entity.TASK); + relationship.setRelation(Relationship.MENTIONED_IN.ordinal()); + return relationship; + }) + .toList(); + + if (!relationships.isEmpty()) { + daoCollection.fieldRelationshipDAO().insertMany(relationships); + } } /** From 534683053ef75bb7f44a3f651282caee15dfe1c0 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 16:40:40 -0700 Subject: [PATCH 04/13] refactor(jdbi3): extract shared row/mappers from CollectionDAO (Track 2 finish) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CollectionDAO split left ~12 cross-domain row/mapper carriers (system entity/service counts + pipeline reporting trends/metrics) inside the composite, referenced as CollectionDAO.X from the aggregators and PipelineRepository — the last thing keeping CollectionDAO from being a thin accessor list. Move them into a new leaf interface SharedRowMappers and redirect all 21 references to their canonical SharedRowMappers.X name (no CollectionDAO dependency, breaking the aggregator->CollectionDAO type cycle). CollectionDAO drops 470 -> 150 lines (now just the extends clause, 4 accessors, IntakeFormDAO, AssetDAO). Pure type relocation, no behavior/SQL change; compiles main+test, CollectionDAOCompositionTest passes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/CollectionDAO.java | 326 ---------------- .../service/jdbi3/PipelineRepository.java | 18 +- .../service/jdbi3/SharedRowMappers.java | 349 ++++++++++++++++++ .../service/jdbi3/SystemTokenDAOs.java | 4 +- .../service/jdbi3/TimeSeriesDAOs.java | 20 +- 5 files changed, 370 insertions(+), 347 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SharedRowMappers.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 84d1e1b2e055..c241394d2307 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -16,18 +16,12 @@ import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL; import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.List; -import org.jdbi.v3.core.mapper.RowMapper; -import org.jdbi.v3.core.statement.StatementContext; import org.jdbi.v3.sqlobject.CreateSqlObject; import org.jdbi.v3.sqlobject.customizer.Bind; import org.jdbi.v3.sqlobject.statement.SqlQuery; import org.jdbi.v3.sqlobject.statement.SqlUpdate; import org.openmetadata.schema.entity.governance.IntakeForm; -import org.openmetadata.schema.util.EntitiesCount; -import org.openmetadata.schema.util.ServicesCount; import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery; import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate; import org.openmetadata.service.util.jdbi.BindFQN; @@ -62,326 +56,6 @@ public interface CollectionDAO @CreateSqlObject IntakeFormDAO intakeFormDAO(); - class EntitiesCountRowMapper implements RowMapper { - @Override - public EntitiesCount map(ResultSet rs, StatementContext ctx) throws SQLException { - return new EntitiesCount() - .withTableCount(rs.getInt("tableCount")) - .withTopicCount(rs.getInt("topicCount")) - .withDashboardCount(rs.getInt("dashboardCount")) - .withPipelineCount(rs.getInt("pipelineCount")) - .withMlmodelCount(rs.getInt("mlmodelCount")) - .withServicesCount(rs.getInt("servicesCount")) - .withUserCount(rs.getInt("userCount")) - .withTeamCount(rs.getInt("teamCount")) - .withTestSuiteCount(rs.getInt("testSuiteCount")) - .withStorageContainerCount(rs.getInt("storageContainerCount")) - .withGlossaryCount(rs.getInt("glossaryCount")) - .withGlossaryTermCount(rs.getInt("glossaryTermCount")); - } - } - - class ServicesCountRowMapper implements RowMapper { - @Override - public ServicesCount map(ResultSet rs, StatementContext ctx) throws SQLException { - return new ServicesCount() - .withDatabaseServiceCount(rs.getInt("databaseServiceCount")) - .withMessagingServiceCount(rs.getInt("messagingServiceCount")) - .withDashboardServiceCount(rs.getInt("dashboardServiceCount")) - .withPipelineServiceCount(rs.getInt("pipelineServiceCount")) - .withMlModelServiceCount(rs.getInt("mlModelServiceCount")) - .withStorageServiceCount(rs.getInt("storageServiceCount")); - } - } - - class ExecutionTrendRow { - private String dateKey; - private String status; - private Integer count; - - public ExecutionTrendRow() {} - - public ExecutionTrendRow(String dateKey, String status, Integer count) { - this.dateKey = dateKey; - this.status = status; - this.count = count; - } - - public String getDateKey() { - return dateKey; - } - - public void setDateKey(String dateKey) { - this.dateKey = dateKey; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public Integer getCount() { - return count; - } - - public void setCount(Integer count) { - this.count = count; - } - } - - class ExecutionTrendRowMapper implements RowMapper { - @Override - public ExecutionTrendRow map(ResultSet rs, StatementContext ctx) throws SQLException { - ExecutionTrendRow row = new ExecutionTrendRow(); - row.setDateKey(rs.getString("date_key")); - row.setStatus(rs.getString("status")); - row.setCount(rs.getInt("count")); - return row; - } - } - - class RuntimeTrendRow { - private String dateKey; - private Long firstTimestamp; - private Double maxRuntime; - private Double minRuntime; - private Double avgRuntime; - private Integer totalPipelines; - - public RuntimeTrendRow() {} - - public RuntimeTrendRow( - String dateKey, - Long firstTimestamp, - Double maxRuntime, - Double minRuntime, - Double avgRuntime, - Integer totalPipelines) { - this.dateKey = dateKey; - this.firstTimestamp = firstTimestamp; - this.maxRuntime = maxRuntime; - this.minRuntime = minRuntime; - this.avgRuntime = avgRuntime; - this.totalPipelines = totalPipelines; - } - - public String getDateKey() { - return dateKey; - } - - public void setDateKey(String dateKey) { - this.dateKey = dateKey; - } - - public Long getFirstTimestamp() { - return firstTimestamp; - } - - public void setFirstTimestamp(Long firstTimestamp) { - this.firstTimestamp = firstTimestamp; - } - - public Double getMaxRuntime() { - return maxRuntime; - } - - public void setMaxRuntime(Double maxRuntime) { - this.maxRuntime = maxRuntime; - } - - public Double getMinRuntime() { - return minRuntime; - } - - public void setMinRuntime(Double minRuntime) { - this.minRuntime = minRuntime; - } - - public Double getAvgRuntime() { - return avgRuntime; - } - - public void setAvgRuntime(Double avgRuntime) { - this.avgRuntime = avgRuntime; - } - - public Integer getTotalPipelines() { - return totalPipelines; - } - - public void setTotalPipelines(Integer totalPipelines) { - this.totalPipelines = totalPipelines; - } - } - - class RuntimeTrendRowMapper implements RowMapper { - @Override - public RuntimeTrendRow map(ResultSet rs, StatementContext ctx) throws SQLException { - RuntimeTrendRow row = new RuntimeTrendRow(); - row.setDateKey(rs.getString("date_key")); - row.setFirstTimestamp(rs.getLong("first_timestamp")); - row.setMaxRuntime(rs.getDouble("max_runtime")); - row.setMinRuntime(rs.getDouble("min_runtime")); - row.setAvgRuntime(rs.getDouble("avg_runtime")); - row.setTotalPipelines(rs.getInt("total_pipelines")); - return row; - } - } - - class ServiceBreakdownRow { - private String serviceType; - private Integer pipelineCount; - - public ServiceBreakdownRow() {} - - public ServiceBreakdownRow(String serviceType, Integer pipelineCount) { - this.serviceType = serviceType; - this.pipelineCount = pipelineCount; - } - - public String getServiceType() { - return serviceType; - } - - public void setServiceType(String serviceType) { - this.serviceType = serviceType; - } - - public Integer getPipelineCount() { - return pipelineCount; - } - - public void setPipelineCount(Integer pipelineCount) { - this.pipelineCount = pipelineCount; - } - } - - class ServiceBreakdownRowMapper implements RowMapper { - @Override - public ServiceBreakdownRow map(ResultSet rs, StatementContext ctx) throws SQLException { - ServiceBreakdownRow row = new ServiceBreakdownRow(); - row.setServiceType(rs.getString("service_type")); - row.setPipelineCount(rs.getInt("pipeline_count")); - return row; - } - } - - class PipelineMetricsRow { - private Integer totalPipelines; - private Integer activePipelines; - private Integer successfulPipelines; - private Integer failedPipelines; - - public PipelineMetricsRow() {} - - public PipelineMetricsRow( - Integer totalPipelines, - Integer activePipelines, - Integer successfulPipelines, - Integer failedPipelines) { - this.totalPipelines = totalPipelines; - this.activePipelines = activePipelines; - this.successfulPipelines = successfulPipelines; - this.failedPipelines = failedPipelines; - } - - public Integer getTotalPipelines() { - return totalPipelines; - } - - public void setTotalPipelines(Integer totalPipelines) { - this.totalPipelines = totalPipelines; - } - - public Integer getActivePipelines() { - return activePipelines; - } - - public void setActivePipelines(Integer activePipelines) { - this.activePipelines = activePipelines; - } - - public Integer getSuccessfulPipelines() { - return successfulPipelines; - } - - public void setSuccessfulPipelines(Integer successfulPipelines) { - this.successfulPipelines = successfulPipelines; - } - - public Integer getFailedPipelines() { - return failedPipelines; - } - - public void setFailedPipelines(Integer failedPipelines) { - this.failedPipelines = failedPipelines; - } - } - - class PipelineMetricsRowMapper implements RowMapper { - @Override - public PipelineMetricsRow map(ResultSet rs, StatementContext ctx) throws SQLException { - PipelineMetricsRow row = new PipelineMetricsRow(); - row.setTotalPipelines(rs.getInt("total_pipelines")); - row.setActivePipelines(rs.getInt("active_pipelines")); - row.setSuccessfulPipelines(rs.getInt("successful_pipelines")); - row.setFailedPipelines(rs.getInt("failed_pipelines")); - return row; - } - } - - class PipelineSummaryRow { - private String id; - private String json; - private String latestStatus; - - public PipelineSummaryRow() {} - - public PipelineSummaryRow(String id, String json, String latestStatus) { - this.id = id; - this.json = json; - this.latestStatus = latestStatus; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getJson() { - return json; - } - - public void setJson(String json) { - this.json = json; - } - - public String getLatestStatus() { - return latestStatus; - } - - public void setLatestStatus(String latestStatus) { - this.latestStatus = latestStatus; - } - } - - class PipelineSummaryRowMapper implements RowMapper { - @Override - public PipelineSummaryRow map(ResultSet rs, StatementContext ctx) throws SQLException { - PipelineSummaryRow row = new PipelineSummaryRow(); - row.setId(rs.getString("id")); - row.setJson(rs.getString("json")); - row.setLatestStatus(rs.getString("latest_status")); - return row; - } - } - interface IntakeFormDAO extends EntityDAO { @Override default String getTableName() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java index 8684fed27e82..c5e178f41bf6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java @@ -1716,7 +1716,7 @@ public ResultList listPipelineSummaries( } // Call database-level filtered query - List rows = + List rows = daoCollection .entityExtensionTimeSeriesDao() .listPipelineSummariesFiltered( @@ -1749,7 +1749,7 @@ public ResultList listPipelineSummaries( // Convert rows to Pipeline objects and build summaries List summaries = new ArrayList<>(); - for (CollectionDAO.PipelineSummaryRow row : rows) { + for (SharedRowMappers.PipelineSummaryRow row : rows) { try { // Parse pipeline JSON Pipeline pipeline = JsonUtils.readValue(row.getJson(), Pipeline.class); @@ -1963,7 +1963,7 @@ private PipelineMetrics getPipelineMetricsFromDB( String startTsFilter = buildStartTsFilter(startTs); String endTsFilter = buildEndTsFilter(endTs); - CollectionDAO.PipelineMetricsRow metricsRow = + SharedRowMappers.PipelineMetricsRow metricsRow = daoCollection .entityExtensionTimeSeriesDao() .getPipelineMetricsData( @@ -1976,7 +1976,7 @@ private PipelineMetrics getPipelineMetricsFromDB( tierFilter, startTsFilter, endTsFilter); - List serviceRows = + List serviceRows = daoCollection .entityExtensionTimeSeriesDao() .getServiceBreakdown( @@ -2003,7 +2003,7 @@ private PipelineMetrics getPipelineMetricsFromDB( List breakdowns = new ArrayList<>(); metrics.setServiceCount(serviceRows.size()); - for (CollectionDAO.ServiceBreakdownRow row : serviceRows) { + for (SharedRowMappers.ServiceBreakdownRow row : serviceRows) { ServiceBreakdown breakdown = new ServiceBreakdown(); breakdown.setServiceType(row.getServiceType()); breakdown.setCount(row.getPipelineCount()); @@ -2095,7 +2095,7 @@ private PipelineExecutionTrendList getPipelineExecutionTrendFromDB( String ownerFilter = buildOwnerFilter(owner); String tierFilter = buildTierFilter(tier); - List rows = + List rows = daoCollection .entityExtensionTimeSeriesDao() .getExecutionTrendData( @@ -2113,7 +2113,7 @@ private PipelineExecutionTrendList getPipelineExecutionTrendFromDB( Map trendMap = new HashMap<>(); int totalSuccess = 0, totalFailed = 0, totalExecutions = 0; - for (CollectionDAO.ExecutionTrendRow row : rows) { + for (SharedRowMappers.ExecutionTrendRow row : rows) { PipelineExecutionTrend trend = trendMap.computeIfAbsent( row.getDateKey(), @@ -2263,7 +2263,7 @@ private PipelineRuntimeTrendList getPipelineRuntimeTrendFromDB( String ownerFilter = buildOwnerFilter(owner); String tierFilter = buildTierFilter(tier); - List rows = + List rows = daoCollection .entityExtensionTimeSeriesDao() .getRuntimeTrendData( @@ -2279,7 +2279,7 @@ private PipelineRuntimeTrendList getPipelineRuntimeTrendFromDB( tierFilter); List trends = new ArrayList<>(); - for (CollectionDAO.RuntimeTrendRow row : rows) { + for (SharedRowMappers.RuntimeTrendRow row : rows) { PipelineRuntimeTrend trend = new PipelineRuntimeTrend(); trend.setDate(row.getDateKey()); trend.setTimestamp(row.getFirstTimestamp()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SharedRowMappers.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SharedRowMappers.java new file mode 100644 index 000000000000..30da8f7c9c20 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SharedRowMappers.java @@ -0,0 +1,349 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import java.sql.ResultSet; +import java.sql.SQLException; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; +import org.openmetadata.schema.util.EntitiesCount; +import org.openmetadata.schema.util.ServicesCount; + +/** + * Cross-domain row/mapper carriers shared by multiple DAO aggregators (system entity/service + * counts and pipeline reporting trends/metrics). Kept out of {@link CollectionDAO} so the + * composite stays a thin set of accessors; referenced by their canonical {@code SharedRowMappers.X} + * name from the DAOs and repositories that build these rows. + */ +public interface SharedRowMappers { + class EntitiesCountRowMapper implements RowMapper { + @Override + public EntitiesCount map(ResultSet rs, StatementContext ctx) throws SQLException { + return new EntitiesCount() + .withTableCount(rs.getInt("tableCount")) + .withTopicCount(rs.getInt("topicCount")) + .withDashboardCount(rs.getInt("dashboardCount")) + .withPipelineCount(rs.getInt("pipelineCount")) + .withMlmodelCount(rs.getInt("mlmodelCount")) + .withServicesCount(rs.getInt("servicesCount")) + .withUserCount(rs.getInt("userCount")) + .withTeamCount(rs.getInt("teamCount")) + .withTestSuiteCount(rs.getInt("testSuiteCount")) + .withStorageContainerCount(rs.getInt("storageContainerCount")) + .withGlossaryCount(rs.getInt("glossaryCount")) + .withGlossaryTermCount(rs.getInt("glossaryTermCount")); + } + } + + class ServicesCountRowMapper implements RowMapper { + @Override + public ServicesCount map(ResultSet rs, StatementContext ctx) throws SQLException { + return new ServicesCount() + .withDatabaseServiceCount(rs.getInt("databaseServiceCount")) + .withMessagingServiceCount(rs.getInt("messagingServiceCount")) + .withDashboardServiceCount(rs.getInt("dashboardServiceCount")) + .withPipelineServiceCount(rs.getInt("pipelineServiceCount")) + .withMlModelServiceCount(rs.getInt("mlModelServiceCount")) + .withStorageServiceCount(rs.getInt("storageServiceCount")); + } + } + + class ExecutionTrendRow { + private String dateKey; + private String status; + private Integer count; + + public ExecutionTrendRow() {} + + public ExecutionTrendRow(String dateKey, String status, Integer count) { + this.dateKey = dateKey; + this.status = status; + this.count = count; + } + + public String getDateKey() { + return dateKey; + } + + public void setDateKey(String dateKey) { + this.dateKey = dateKey; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Integer getCount() { + return count; + } + + public void setCount(Integer count) { + this.count = count; + } + } + + class ExecutionTrendRowMapper implements RowMapper { + @Override + public ExecutionTrendRow map(ResultSet rs, StatementContext ctx) throws SQLException { + ExecutionTrendRow row = new ExecutionTrendRow(); + row.setDateKey(rs.getString("date_key")); + row.setStatus(rs.getString("status")); + row.setCount(rs.getInt("count")); + return row; + } + } + + class RuntimeTrendRow { + private String dateKey; + private Long firstTimestamp; + private Double maxRuntime; + private Double minRuntime; + private Double avgRuntime; + private Integer totalPipelines; + + public RuntimeTrendRow() {} + + public RuntimeTrendRow( + String dateKey, + Long firstTimestamp, + Double maxRuntime, + Double minRuntime, + Double avgRuntime, + Integer totalPipelines) { + this.dateKey = dateKey; + this.firstTimestamp = firstTimestamp; + this.maxRuntime = maxRuntime; + this.minRuntime = minRuntime; + this.avgRuntime = avgRuntime; + this.totalPipelines = totalPipelines; + } + + public String getDateKey() { + return dateKey; + } + + public void setDateKey(String dateKey) { + this.dateKey = dateKey; + } + + public Long getFirstTimestamp() { + return firstTimestamp; + } + + public void setFirstTimestamp(Long firstTimestamp) { + this.firstTimestamp = firstTimestamp; + } + + public Double getMaxRuntime() { + return maxRuntime; + } + + public void setMaxRuntime(Double maxRuntime) { + this.maxRuntime = maxRuntime; + } + + public Double getMinRuntime() { + return minRuntime; + } + + public void setMinRuntime(Double minRuntime) { + this.minRuntime = minRuntime; + } + + public Double getAvgRuntime() { + return avgRuntime; + } + + public void setAvgRuntime(Double avgRuntime) { + this.avgRuntime = avgRuntime; + } + + public Integer getTotalPipelines() { + return totalPipelines; + } + + public void setTotalPipelines(Integer totalPipelines) { + this.totalPipelines = totalPipelines; + } + } + + class RuntimeTrendRowMapper implements RowMapper { + @Override + public RuntimeTrendRow map(ResultSet rs, StatementContext ctx) throws SQLException { + RuntimeTrendRow row = new RuntimeTrendRow(); + row.setDateKey(rs.getString("date_key")); + row.setFirstTimestamp(rs.getLong("first_timestamp")); + row.setMaxRuntime(rs.getDouble("max_runtime")); + row.setMinRuntime(rs.getDouble("min_runtime")); + row.setAvgRuntime(rs.getDouble("avg_runtime")); + row.setTotalPipelines(rs.getInt("total_pipelines")); + return row; + } + } + + class ServiceBreakdownRow { + private String serviceType; + private Integer pipelineCount; + + public ServiceBreakdownRow() {} + + public ServiceBreakdownRow(String serviceType, Integer pipelineCount) { + this.serviceType = serviceType; + this.pipelineCount = pipelineCount; + } + + public String getServiceType() { + return serviceType; + } + + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + + public Integer getPipelineCount() { + return pipelineCount; + } + + public void setPipelineCount(Integer pipelineCount) { + this.pipelineCount = pipelineCount; + } + } + + class ServiceBreakdownRowMapper implements RowMapper { + @Override + public ServiceBreakdownRow map(ResultSet rs, StatementContext ctx) throws SQLException { + ServiceBreakdownRow row = new ServiceBreakdownRow(); + row.setServiceType(rs.getString("service_type")); + row.setPipelineCount(rs.getInt("pipeline_count")); + return row; + } + } + + class PipelineMetricsRow { + private Integer totalPipelines; + private Integer activePipelines; + private Integer successfulPipelines; + private Integer failedPipelines; + + public PipelineMetricsRow() {} + + public PipelineMetricsRow( + Integer totalPipelines, + Integer activePipelines, + Integer successfulPipelines, + Integer failedPipelines) { + this.totalPipelines = totalPipelines; + this.activePipelines = activePipelines; + this.successfulPipelines = successfulPipelines; + this.failedPipelines = failedPipelines; + } + + public Integer getTotalPipelines() { + return totalPipelines; + } + + public void setTotalPipelines(Integer totalPipelines) { + this.totalPipelines = totalPipelines; + } + + public Integer getActivePipelines() { + return activePipelines; + } + + public void setActivePipelines(Integer activePipelines) { + this.activePipelines = activePipelines; + } + + public Integer getSuccessfulPipelines() { + return successfulPipelines; + } + + public void setSuccessfulPipelines(Integer successfulPipelines) { + this.successfulPipelines = successfulPipelines; + } + + public Integer getFailedPipelines() { + return failedPipelines; + } + + public void setFailedPipelines(Integer failedPipelines) { + this.failedPipelines = failedPipelines; + } + } + + class PipelineMetricsRowMapper implements RowMapper { + @Override + public PipelineMetricsRow map(ResultSet rs, StatementContext ctx) throws SQLException { + PipelineMetricsRow row = new PipelineMetricsRow(); + row.setTotalPipelines(rs.getInt("total_pipelines")); + row.setActivePipelines(rs.getInt("active_pipelines")); + row.setSuccessfulPipelines(rs.getInt("successful_pipelines")); + row.setFailedPipelines(rs.getInt("failed_pipelines")); + return row; + } + } + + class PipelineSummaryRow { + private String id; + private String json; + private String latestStatus; + + public PipelineSummaryRow() {} + + public PipelineSummaryRow(String id, String json, String latestStatus) { + this.id = id; + this.json = json; + this.latestStatus = latestStatus; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getJson() { + return json; + } + + public void setJson(String json) { + this.json = json; + } + + public String getLatestStatus() { + return latestStatus; + } + + public void setLatestStatus(String latestStatus) { + this.latestStatus = latestStatus; + } + } + + class PipelineSummaryRowMapper implements RowMapper { + @Override + public PipelineSummaryRow map(ResultSet rs, StatementContext ctx) throws SQLException { + PipelineSummaryRow row = new PipelineSummaryRow(); + row.setId(rs.getString("id")); + row.setJson(rs.getString("json")); + row.setLatestStatus(rs.getString("latest_status")); + return row; + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemTokenDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemTokenDAOs.java index 71eebdeb2299..4e1ead35b006 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemTokenDAOs.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemTokenDAOs.java @@ -122,7 +122,7 @@ interface SystemDAO { + "(SELECT COUNT(*) FROM team_entity ) as teamCount, " + "(SELECT COUNT(*) FROM test_suite ) as testSuiteCount", connectionType = POSTGRES) - @RegisterRowMapper(CollectionDAO.EntitiesCountRowMapper.class) + @RegisterRowMapper(SharedRowMappers.EntitiesCountRowMapper.class) EntitiesCount getAggregatedEntitiesCount(@Define("cond") String cond) throws StatementException; @ConnectionAwareSqlQuery( @@ -145,7 +145,7 @@ interface SystemDAO { + "(SELECT COUNT(*) FROM storage_service_entity ) as storageServiceCount, " + "(SELECT COUNT(*) FROM search_service_entity ) as searchServiceCount", connectionType = POSTGRES) - @RegisterRowMapper(CollectionDAO.ServicesCountRowMapper.class) + @RegisterRowMapper(SharedRowMappers.ServicesCountRowMapper.class) ServicesCount getAggregatedServicesCount(@Define("cond") String cond) throws StatementException; @SqlQuery("SELECT configType,json FROM openmetadata_settings") diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java index 40db61308221..4589b0801e51 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java @@ -661,8 +661,8 @@ default String getTimeSeriesTableName() { + "GROUP BY date_key, status " + "ORDER BY date_key ASC", connectionType = POSTGRES) - @RegisterRowMapper(CollectionDAO.ExecutionTrendRowMapper.class) - List getExecutionTrendData( + @RegisterRowMapper(SharedRowMappers.ExecutionTrendRowMapper.class) + List getExecutionTrendData( @Bind("startTs") Long startTs, @Bind("endTs") Long endTs, @Define("pipelineFqnFilter") String pipelineFqnFilter, @@ -774,8 +774,8 @@ List getExecutionTrendData( + "GROUP BY date_key " + "ORDER BY date_key ASC", connectionType = POSTGRES) - @RegisterRowMapper(CollectionDAO.RuntimeTrendRowMapper.class) - List getRuntimeTrendData( + @RegisterRowMapper(SharedRowMappers.RuntimeTrendRowMapper.class) + List getRuntimeTrendData( @Bind("startTs") Long startTs, @Bind("endTs") Long endTs, @Define("pipelineFqnFilter") String pipelineFqnFilter, @@ -827,8 +827,8 @@ List getRuntimeTrendData( + " " + "GROUP BY service_type", connectionType = POSTGRES) - @RegisterRowMapper(CollectionDAO.ServiceBreakdownRowMapper.class) - List getServiceBreakdown( + @RegisterRowMapper(SharedRowMappers.ServiceBreakdownRowMapper.class) + List getServiceBreakdown( @Define("serviceTypeFilter") String serviceTypeFilter, @Define("serviceFilter") String serviceFilter, @Define("mysqlStatusFilter") String mysqlStatusFilter, @@ -881,8 +881,8 @@ List getServiceBreakdown( + " " + " ", connectionType = POSTGRES) - @RegisterRowMapper(CollectionDAO.PipelineMetricsRowMapper.class) - CollectionDAO.PipelineMetricsRow getPipelineMetricsData( + @RegisterRowMapper(SharedRowMappers.PipelineMetricsRowMapper.class) + SharedRowMappers.PipelineMetricsRow getPipelineMetricsData( @Define("serviceTypeFilter") String serviceTypeFilter, @Define("serviceFilter") String serviceFilter, @Define("mysqlStatusFilter") String mysqlStatusFilter, @@ -931,8 +931,8 @@ CollectionDAO.PipelineMetricsRow getPipelineMetricsData( + "ORDER BY pe.name " + "LIMIT :limit OFFSET :offset", connectionType = POSTGRES) - @RegisterRowMapper(CollectionDAO.PipelineSummaryRowMapper.class) - List listPipelineSummariesFiltered( + @RegisterRowMapper(SharedRowMappers.PipelineSummaryRowMapper.class) + List listPipelineSummariesFiltered( @Define("serviceFilter") String serviceFilter, @Define("mysqlServiceTypeFilter") String mysqlServiceTypeFilter, @Define("postgresServiceTypeFilter") String postgresServiceTypeFilter, From e32736f696a524dfd8ac591752c01ce784624129 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 18:11:11 -0700 Subject: [PATCH 05/13] fix(it): redirect moved jdbi3 symbols in integration-tests after the CollectionDAO/EntityRepository split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CollectionDAO/EntityRepository decomposition moved member types into aggregator interfaces and static caches into EntityCaches, but the redirect never covered the openmetadata-integration-tests module, breaking its compile: - SearchIndexRetryQueueIT imported CollectionDAO.SearchIndexRetryQueueDAO; that type now lives in SearchReindexDAOs, and a single-type import requires the canonical declaring interface (inherited member types can't be imported via a subtype). Repointed both imports to SearchReindexDAOs. - EntityCacheInvalidationIT referenced EntityRepository.CACHE_WITH_ID; that field moved to EntityCaches (static fields aren't inherited via the subclass name). The remaining "cannot find symbol: log" errors CI reported were Lombok cascade fallout from these two — they clear once the real errors are fixed. Integration -tests module now compiles (BUILD SUCCESS). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../openmetadata/it/tests/EntityCacheInvalidationIT.java | 9 ++++----- .../openmetadata/it/tests/SearchIndexRetryQueueIT.java | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/EntityCacheInvalidationIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/EntityCacheInvalidationIT.java index bf1ee94aa624..876b82daedfe 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/EntityCacheInvalidationIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/EntityCacheInvalidationIT.java @@ -31,7 +31,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.EntityCaches; import org.openmetadata.service.util.RequestEntityCache; /** @@ -124,11 +124,11 @@ void testGuavaCacheEntryRemovedAfterUpdate(TestNamespace ns) throws Exception { UUID domainId = domain.getId(); // Force a cache load - EntityRepository.CACHE_WITH_ID.get(new ImmutablePair<>(Entity.DOMAIN, domainId)); + EntityCaches.CACHE_WITH_ID.get(new ImmutablePair<>(Entity.DOMAIN, domainId)); // Verify cache has the entity String cachedJson = - EntityRepository.CACHE_WITH_ID.getIfPresent(new ImmutablePair<>(Entity.DOMAIN, domainId)); + EntityCaches.CACHE_WITH_ID.getIfPresent(new ImmutablePair<>(Entity.DOMAIN, domainId)); assertNotNull(cachedJson, "Cache should contain the entity after get()"); Domain cachedEntity = JsonUtils.readValue(cachedJson, Domain.class); assertEquals("before update", cachedEntity.getDescription()); @@ -138,8 +138,7 @@ void testGuavaCacheEntryRemovedAfterUpdate(TestNamespace ns) throws Exception { SdkClients.adminClient().domains().update(domainId.toString(), domain); // After update, re-loading from cache should get fresh data - String freshJson = - EntityRepository.CACHE_WITH_ID.get(new ImmutablePair<>(Entity.DOMAIN, domainId)); + String freshJson = EntityCaches.CACHE_WITH_ID.get(new ImmutablePair<>(Entity.DOMAIN, domainId)); Domain freshEntity = JsonUtils.readValue(freshJson, Domain.class); assertEquals( "after update", diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexRetryQueueIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexRetryQueueIT.java index ba39079e44da..3aef8c4691ee 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexRetryQueueIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchIndexRetryQueueIT.java @@ -28,8 +28,8 @@ import org.openmetadata.schema.entity.data.Table; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexRetryQueueDAO; -import org.openmetadata.service.jdbi3.CollectionDAO.SearchIndexRetryQueueDAO.SearchIndexRetryRecord; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexRetryQueueDAO; +import org.openmetadata.service.jdbi3.SearchReindexDAOs.SearchIndexRetryQueueDAO.SearchIndexRetryRecord; import org.openmetadata.service.search.SearchIndexRetryQueue; import org.openmetadata.service.search.SearchIndexRetryWorker; import org.openmetadata.service.search.SearchRepository; From e9fa5c9b25854b65507181dd9ae30b22fa7e5973 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 18:11:22 -0700 Subject: [PATCH 06/13] fix(jdbi3): make tag_usage cleanup keyset pagination NULL-safe (E1 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial audit (proven on a live MySQL container) found the E1 keyset cleanup could silently skip rows: tagFQNHash/targetFQNHash are nullable, and once the cursor lands on a NULL-hash row (MySQL sorts NULLs first, so this is reached early), every subsequent `tagFQNHash > NULL` / `= NULL` / row-constructor compare is UNKNOWN, the batch returns empty, and the scan terminates — skipping all trailing rows. The two engine forms also diverge on NULL ordering. Fix: exclude NULL hashes from the keyset scan (AND tagFQNHash IS NOT NULL AND targetFQNHash IS NOT NULL) so the cursor only ever holds real keys. This keeps the index seek (verified: MySQL type=range, Postgres Index Only Scan) and makes both engines scan an identical row set regardless of their NULL ordering. The rare malformed NULL-hash rows (no write path produces them) are swept once via a new bounded getTagUsagesWithNullHash query so the cleanup tool still sees them. Validated on MySQL 8.0 + Postgres 16: with NULL rows present, a batchSize=2 walk now covers all non-NULL rows (0 dropped, 0 dupes) and the NULL sweep catches the malformed rows; the pre-fix walk terminated early. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/ClassificationTagDAOs.java | 26 ++++++++++++- .../service/util/TagUsageCleanup.java | 37 ++++++++++++++----- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java index ca4434767eba..0dfa1e6608dd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationTagDAOs.java @@ -1157,13 +1157,23 @@ void deleteTagsBatchInternal( // OR form is required on MySQL — its optimizer turns it into an index range seek, whereas a // row-constructor comparison scans the whole index from the start (verified via // Handler_read_next). Postgres seeks optimally with the row-constructor (Index Only Scan). + // The tagFQNHash/targetFQNHash columns are nullable, and comparing a cursor against a NULL hash + // yields UNKNOWN — which would silently terminate the scan and skip the remaining rows once the + // cursor lands on a NULL-hash row (MySQL sorts NULLs first, so this is reachable early). + // Excluding + // NULL hashes here keeps the cursor on real keys (preserving the index seek) and makes both + // engine + // forms scan an identical row set regardless of their differing NULL ordering; the rare + // malformed + // NULL-hash rows are swept separately via getTagUsagesWithNullHash. @ConnectionAwareSqlQuery( value = "SELECT source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedAt, appliedBy, metadata " + "FROM tag_usage " - + "WHERE source > :lastSource " + + "WHERE (source > :lastSource " + "OR (source = :lastSource AND tagFQNHash > :lastTagFQNHash) " - + "OR (source = :lastSource AND tagFQNHash = :lastTagFQNHash AND targetFQNHash > :lastTargetFQNHash) " + + "OR (source = :lastSource AND tagFQNHash = :lastTagFQNHash AND targetFQNHash > :lastTargetFQNHash)) " + + "AND tagFQNHash IS NOT NULL AND targetFQNHash IS NOT NULL " + "ORDER BY source, tagFQNHash, targetFQNHash LIMIT :limit", connectionType = MYSQL) @ConnectionAwareSqlQuery( @@ -1171,6 +1181,7 @@ void deleteTagsBatchInternal( "SELECT source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedAt, appliedBy, metadata " + "FROM tag_usage " + "WHERE (source, tagFQNHash, targetFQNHash) > (:lastSource, :lastTagFQNHash, :lastTargetFQNHash) " + + "AND tagFQNHash IS NOT NULL AND targetFQNHash IS NOT NULL " + "ORDER BY source, tagFQNHash, targetFQNHash LIMIT :limit", connectionType = POSTGRES) @RegisterRowMapper(TagUsageObjectMapper.class) @@ -1180,6 +1191,17 @@ List getTagUsagesAfter( @Bind("lastTargetFQNHash") String lastTargetFQNHash, @Bind("limit") int limit); + // Malformed rows whose hash columns are NULL cannot be keyset-ordered, so they are swept in one + // bounded query. No current write path produces NULL hashes (FullyQualifiedName.buildHash only + // returns null for null/empty FQNs), so this is expected to return nothing — but the cleanup + // tool + // exists precisely to remove malformed rows, so it must still see them. + @SqlQuery( + "SELECT source, tagFQN, tagFQNHash, targetFQNHash, labelType, state, reason, appliedAt, appliedBy, metadata " + + "FROM tag_usage WHERE tagFQNHash IS NULL OR targetFQNHash IS NULL LIMIT :limit") + @RegisterRowMapper(TagUsageObjectMapper.class) + List getTagUsagesWithNullHash(@Bind("limit") int limit); + @SqlUpdate( "DELETE FROM tag_usage WHERE source = :source AND tagFQNHash = :tagFQNHash AND targetFQNHash = :targetFQNHash") int deleteTagUsage( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/TagUsageCleanup.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/TagUsageCleanup.java index 5bc8a9272999..bb24742a225c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/TagUsageCleanup.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/TagUsageCleanup.java @@ -33,6 +33,11 @@ @Slf4j public class TagUsageCleanup { + // Malformed tag_usage rows have NULL hash columns that cannot be keyset-paginated; they are swept + // in one bounded query. Expected to be empty (no write path produces NULL hashes), so a generous + // cap is safe; exceeding it is logged so an operator can investigate a pathological table. + private static final int MAX_NULL_HASH_ROWS = 10_000; + private final CollectionDAO collectionDAO; private final TagRepository tagRepository; private final GlossaryTermRepository glossaryTermRepository; @@ -91,7 +96,7 @@ public TagCleanupResult performCleanup(int batchSize) { int lastSource = -1; String lastTagFQNHash = ""; String lastTargetFQNHash = ""; - int processedCount = 0; + int processedCount = sweepNullHashTagUsages(result); int batchNumber = 1; boolean hasMore = true; @@ -111,14 +116,8 @@ public TagCleanupResult performCleanup(int batchSize) { LOG.info("No more tag usages to process"); hasMore = false; } else { - for (CollectionDAO.TagUsageObject tagUsage : tagUsageBatch) { - OrphanedTagUsage orphan = validateTagUsage(tagUsage); - if (orphan != null) { - result.getOrphanedTagUsages().add(orphan); - result.getOrphansBySource().merge(orphan.getSourceName(), 1, Integer::sum); - } - processedCount++; - } + tagUsageBatch.forEach(tagUsage -> processTagUsage(tagUsage, result)); + processedCount += tagUsageBatch.size(); CollectionDAO.TagUsageObject lastRow = tagUsageBatch.getLast(); lastSource = lastRow.getSource(); @@ -163,6 +162,26 @@ public TagCleanupResult performCleanup(int batchSize) { return result; } + private int sweepNullHashTagUsages(TagCleanupResult result) { + List nullHashRows = + collectionDAO.tagUsageDAO().getTagUsagesWithNullHash(MAX_NULL_HASH_ROWS); + nullHashRows.forEach(tagUsage -> processTagUsage(tagUsage, result)); + if (nullHashRows.size() == MAX_NULL_HASH_ROWS) { + LOG.warn( + "Reached the {}-row cap while sweeping tag_usage rows with NULL hashes; some malformed rows may remain. Investigate and re-run cleanup.", + MAX_NULL_HASH_ROWS); + } + return nullHashRows.size(); + } + + private void processTagUsage(CollectionDAO.TagUsageObject tagUsage, TagCleanupResult result) { + OrphanedTagUsage orphan = validateTagUsage(tagUsage); + if (orphan != null) { + result.getOrphanedTagUsages().add(orphan); + result.getOrphansBySource().merge(orphan.getSourceName(), 1, Integer::sum); + } + } + private OrphanedTagUsage validateTagUsage(CollectionDAO.TagUsageObject tagUsage) { try { int source = tagUsage.getSource(); From 460814860db0eb1562a7dd82496e3746314f5451 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 18:31:51 -0700 Subject: [PATCH 07/13] refactor(jdbi3): extract static custom-property validators into CustomPropertyValidator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial architecture review of the remaining EntityRepository clusters found the big ones (EntityUpdater, inheritance/lifecycle template hooks, CSV stubs) must stay — they are polymorphic seams 20-63 subclasses override, and extracting them would force pervasive r.hook() callbacks. The one genuinely clean, callback-free win all three reviewers named independently: the static extension/custom-property validators (~210 lines), which carry zero entity-instance state and whose siblings (validateCustomPropertyEntityReference[List]) already live in EntityUtil. Move validateExtension(Object,String), validateAndTransformExtension, validateHyperlinkUrl, getFormattedDateTimeField, validateTableType, validateEnumKeys to a new org.openmetadata.service.util.CustomPropertyValidator. EntityRepository keeps the instance validateExtension(T,boolean) create/update hook, now delegating to it. Redirected the two external callers (ColumnRepository, EntityCsv) and dropped the two now-unused static imports. EntityRepository -213 lines; pure move, no behavior change. Service (main+test) and integration-tests modules both compile. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../java/org/openmetadata/csv/EntityCsv.java | 4 +- .../service/jdbi3/ColumnRepository.java | 3 +- .../service/jdbi3/EntityRepository.java | 223 +-------------- .../service/util/CustomPropertyValidator.java | 254 ++++++++++++++++++ 4 files changed, 264 insertions(+), 220 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/util/CustomPropertyValidator.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index fcc0fd34e6e1..eaeb51554e27 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -107,6 +107,7 @@ import org.openmetadata.service.jdbi3.TableRepository; import org.openmetadata.service.rules.RuleEngine; import org.openmetadata.service.util.AsyncService; +import org.openmetadata.service.util.CustomPropertyValidator; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.RestUtil.PutResponse; @@ -848,7 +849,8 @@ private Object parseEnumType( String propertyConfig) { List enumKeys = listOrEmpty(fieldToInternalArray(fieldValue.toString())); try { - EntityRepository.validateEnumKeys(fieldName, JsonUtils.valueToTree(enumKeys), propertyConfig); + CustomPropertyValidator.validateEnumKeys( + fieldName, JsonUtils.valueToTree(enumKeys), propertyConfig); } catch (Exception e) { deferredFailure( csvRecord, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java index 0b08376b5262..324c135badbb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java @@ -74,6 +74,7 @@ import org.openmetadata.service.security.policyevaluator.OperationContext; import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.security.policyevaluator.ResourceContextInterface; +import org.openmetadata.service.util.CustomPropertyValidator; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.RestUtil; @@ -366,7 +367,7 @@ private void applyColumnUpdates( .ifPresent( ext -> { Object transformedExtension = - EntityRepository.validateAndTransformExtension(ext, columnEntityType); + CustomPropertyValidator.validateAndTransformExtension(ext, columnEntityType); column.setExtension(transformedExtension); }); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 5f47a2df65fb..9baf5fa1860f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -78,8 +78,6 @@ import static org.openmetadata.service.util.EntityUtil.nextVersion; import static org.openmetadata.service.util.EntityUtil.objectMatch; import static org.openmetadata.service.util.EntityUtil.tagLabelMatch; -import static org.openmetadata.service.util.EntityUtil.validateCustomPropertyEntityReference; -import static org.openmetadata.service.util.EntityUtil.validateCustomPropertyEntityReferenceList; import static org.openmetadata.service.util.LineageUtil.addDataProductsLineage; import static org.openmetadata.service.util.LineageUtil.addDomainLineage; import static org.openmetadata.service.util.LineageUtil.removeDataProductsLineage; @@ -93,11 +91,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.UncheckedExecutionException; -import com.networknt.schema.Error; -import com.networknt.schema.Schema; import io.micrometer.core.instrument.Metrics; import jakarta.json.JsonPatch; -import jakarta.validation.ConstraintViolationException; import jakarta.ws.rs.core.Response.Status; import jakarta.ws.rs.core.SecurityContext; import jakarta.ws.rs.core.UriInfo; @@ -105,12 +100,8 @@ import java.net.URI; import java.time.Instant; import java.time.LocalDateTime; -import java.time.LocalTime; import java.time.Period; import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.time.temporal.TemporalAccessor; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -119,7 +110,6 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; @@ -191,8 +181,6 @@ import org.openmetadata.schema.type.change.ChangeSource; import org.openmetadata.schema.type.change.ChangeSummary; import org.openmetadata.schema.type.csv.CsvImportResult; -import org.openmetadata.schema.type.customProperties.EnumConfig; -import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.schema.utils.ResultList; import org.openmetadata.service.Entity; @@ -234,6 +222,7 @@ import org.openmetadata.service.security.AuthorizationException; import org.openmetadata.service.security.policyevaluator.PolicyEvaluator; import org.openmetadata.service.security.policyevaluator.SubjectContext; +import org.openmetadata.service.util.CustomPropertyValidator; import org.openmetadata.service.util.EntityETag; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.Fields; @@ -4502,220 +4491,17 @@ public final void deleteExtensionBeforeTimestamp(String fqn, String extension, L * This method can be used by other repositories that need to validate extensions * without extending EntityRepository. */ - public static void validateExtension(Object extension, String entityTypeName) { - if (extension == null) { - return; - } - - JsonNode jsonNode = JsonUtils.valueToTree(extension); - Iterator> customFields = jsonNode.fields(); - - while (customFields.hasNext()) { - Entry entry = customFields.next(); - String fieldName = entry.getKey(); - JsonNode fieldValue = entry.getValue(); - - // Validate that the custom property exists for this entity type - Schema jsonSchema = TypeRegistry.instance().getSchema(entityTypeName, fieldName); - if (jsonSchema == null) { - throw new IllegalArgumentException(CatalogExceptionMessage.unknownCustomField(fieldName)); - } - - // Validate against JSON schema - this handles all validation including type-specific rules - List validationMessages = jsonSchema.validate(fieldValue); - if (!validationMessages.isEmpty()) { - throw new IllegalArgumentException( - CatalogExceptionMessage.jsonValidationError(fieldName, validationMessages.toString())); - } - } - } - - public static Object validateAndTransformExtension(Object extension, String entityTypeName) { - if (extension == null) { - return null; - } - - // Validate custom properties existence and schema compliance - validateExtension(extension, entityTypeName); - - // Apply property type-specific transformations (date formatting, enum sorting, etc.) - JsonNode extensionNode = JsonUtils.valueToTree(extension); - if (!extensionNode.isObject()) { - return null; - } - ObjectNode jsonNode = (ObjectNode) extensionNode; - Iterator> customFields = jsonNode.fields(); - - while (customFields.hasNext()) { - Entry entry = customFields.next(); - String fieldName = entry.getKey(); - JsonNode fieldValue = entry.getValue(); - - String customPropertyType = TypeRegistry.getCustomPropertyType(entityTypeName, fieldName); - String propertyConfig = TypeRegistry.getCustomPropertyConfig(entityTypeName, fieldName); - - switch (customPropertyType) { - case "date-cp", "dateTime-cp", "time-cp" -> { - String formattedValue = - getFormattedDateTimeField( - fieldValue.textValue(), customPropertyType, propertyConfig, fieldName); - jsonNode.put(fieldName, formattedValue); - } - case "table-cp" -> validateTableType(fieldValue, propertyConfig, fieldName); - case "enum" -> { - validateEnumKeys(fieldName, fieldValue, propertyConfig); - List enumValues = - StreamSupport.stream(fieldValue.spliterator(), false) - .map(JsonNode::asText) - .sorted() - .collect(Collectors.toList()); - jsonNode.set(fieldName, JsonUtils.valueToTree(enumValues)); - } - case "hyperlink-cp" -> validateHyperlinkUrl(fieldValue, fieldName); - case "entityReference" -> validateCustomPropertyEntityReference(fieldValue, fieldName); - case "entityReferenceList" -> validateCustomPropertyEntityReferenceList( - fieldValue, fieldName); - default -> {} - } - } - - return JsonUtils.treeToValue(jsonNode, Object.class); - } - private void validateExtension(T entity, boolean update) { // Validate complete extension field only on POST if (entity.getExtension() == null || update) { return; } - Object transformedExtension = validateAndTransformExtension(entity.getExtension(), entityType); + Object transformedExtension = + CustomPropertyValidator.validateAndTransformExtension(entity.getExtension(), entityType); entity.setExtension(transformedExtension); } - private static void validateHyperlinkUrl(JsonNode fieldValue, String fieldName) { - if (fieldValue == null || fieldValue.isNull()) { - return; - } - JsonNode urlNode = fieldValue.get("url"); - if (urlNode == null || urlNode.isNull() || urlNode.asText().isEmpty()) { - return; - } - String url = urlNode.asText(); - try { - java.net.URI uri = new java.net.URI(url); - String scheme = uri.getScheme(); - if (scheme == null - || (!scheme.equalsIgnoreCase("http") && !scheme.equalsIgnoreCase("https"))) { - throw new IllegalArgumentException( - String.format( - "Invalid URL protocol for field '%s': URL must use http or https protocol", - fieldName)); - } - } catch (java.net.URISyntaxException e) { - throw new IllegalArgumentException( - String.format("Invalid URL format for field '%s': %s", fieldName, e.getMessage())); - } - } - - private static String getFormattedDateTimeField( - String fieldValue, String customPropertyType, String propertyConfig, String fieldName) { - DateTimeFormatter formatter; - - try { - return switch (customPropertyType) { - case "date-cp" -> { - DateTimeFormatter inputFormatter = - DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH); - TemporalAccessor date = inputFormatter.parse(fieldValue); - DateTimeFormatter outputFormatter = - DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH); - yield outputFormatter.format(date); - } - case "dateTime-cp" -> { - formatter = DateTimeFormatter.ofPattern(propertyConfig); - LocalDateTime dateTime = LocalDateTime.parse(fieldValue, formatter); - yield dateTime.format(formatter); - } - case "time-cp" -> { - formatter = DateTimeFormatter.ofPattern(propertyConfig); - LocalTime time = LocalTime.parse(fieldValue, formatter); - yield time.format(formatter); - } - default -> throw new IllegalArgumentException( - "Unsupported customPropertyType: " + customPropertyType); - }; - } catch (DateTimeParseException e) { - throw new IllegalArgumentException( - CatalogExceptionMessage.dateTimeValidationError(fieldName, propertyConfig)); - } - } - - private static void validateTableType( - JsonNode fieldValue, String propertyConfig, String fieldName) { - TableConfig tableConfig = - JsonUtils.convertValue(JsonUtils.readTree(propertyConfig), TableConfig.class); - org.openmetadata.schema.type.customProperties.Table tableValue = - JsonUtils.convertValue( - JsonUtils.readTree(String.valueOf(fieldValue)), - org.openmetadata.schema.type.customProperties.Table.class); - Set configColumns = tableConfig.getColumns(); - - try { - JsonUtils.validateJsonSchema( - tableValue, org.openmetadata.schema.type.customProperties.Table.class); - - Set fieldColumns = new HashSet<>(); - fieldValue.get("columns").forEach(column -> fieldColumns.add(column.asText())); - - Set undefinedColumns = new HashSet<>(fieldColumns); - undefinedColumns.removeAll(configColumns); - if (!undefinedColumns.isEmpty()) { - throw new IllegalArgumentException( - "Expected columns: " - + configColumns - + ", but found undefined columns: " - + undefinedColumns); - } - - Set rowFieldNames = new HashSet<>(); - fieldValue.get("rows").forEach(row -> row.fieldNames().forEachRemaining(rowFieldNames::add)); - - undefinedColumns = new HashSet<>(rowFieldNames); - undefinedColumns.removeAll(configColumns); - if (!undefinedColumns.isEmpty()) { - throw new IllegalArgumentException("Rows contain undefined columns: " + undefinedColumns); - } - } catch (ConstraintViolationException e) { - String validationErrors = - e.getConstraintViolations().stream() - .map(violation -> violation.getPropertyPath() + " " + violation.getMessage()) - .collect(Collectors.joining(", ")); - - throw new IllegalArgumentException( - CatalogExceptionMessage.jsonValidationError(fieldName, validationErrors)); - } - } - - public static void validateEnumKeys( - String fieldName, JsonNode fieldValue, String propertyConfig) { - JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig); - EnumConfig config = JsonUtils.treeToValue(propertyConfigNode, EnumConfig.class); - - if (!config.getMultiSelect() && fieldValue.size() > 1) { - throw new IllegalArgumentException( - String.format("Only one value allowed for non-multiSelect %s property", fieldName)); - } - Set validValues = new HashSet<>(config.getValues()); - Set fieldValues = new HashSet<>(); - fieldValue.forEach(value -> fieldValues.add(value.asText())); - - if (!validValues.containsAll(fieldValues)) { - fieldValues.removeAll(validValues); - throw new IllegalArgumentException( - String.format("Values '%s' not supported for property %s", fieldValues, fieldName)); - } - } - public final void storeExtension(EntityInterface entity) { if (entity.getExtension() == null) { return; @@ -8338,7 +8124,8 @@ private void updateExtension(boolean consolidatingChanges) { singleField.put( field.getKey(), JsonUtils.treeToValue(field.getValue(), Object.class)); Object transformedField = - validateAndTransformExtension(singleField, entityType); + CustomPropertyValidator.validateAndTransformExtension( + singleField, entityType); JsonNode transformedNode = JsonUtils.valueToTree(transformedField); if (transformedNode.isObject()) { extensionNode.set(field.getKey(), transformedNode.get(field.getKey())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/CustomPropertyValidator.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/CustomPropertyValidator.java new file mode 100644 index 000000000000..5c60f1b5422a --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/CustomPropertyValidator.java @@ -0,0 +1,254 @@ +/* + * Copyright 2024 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.util; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.networknt.schema.Error; +import com.networknt.schema.Schema; +import jakarta.validation.ConstraintViolationException; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.TemporalAccessor; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.openmetadata.schema.type.customProperties.EnumConfig; +import org.openmetadata.schema.type.customProperties.TableConfig; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.service.TypeRegistry; +import org.openmetadata.service.exception.CatalogExceptionMessage; + +/** + * Validates and transforms custom-property ("extension") values against the JSON schema and + * type-specific rules registered for an entity type. Extracted from EntityRepository: these are pure + * static validators with no entity-instance state, sitting next to their {@link EntityUtil} + * custom-property-reference siblings. + */ +public final class CustomPropertyValidator { + + private CustomPropertyValidator() {} + + public static void validateExtension(Object extension, String entityTypeName) { + if (extension == null) { + return; + } + + JsonNode jsonNode = JsonUtils.valueToTree(extension); + Iterator> customFields = jsonNode.fields(); + + while (customFields.hasNext()) { + Entry entry = customFields.next(); + String fieldName = entry.getKey(); + JsonNode fieldValue = entry.getValue(); + + // Validate that the custom property exists for this entity type + Schema jsonSchema = TypeRegistry.instance().getSchema(entityTypeName, fieldName); + if (jsonSchema == null) { + throw new IllegalArgumentException(CatalogExceptionMessage.unknownCustomField(fieldName)); + } + + // Validate against JSON schema - this handles all validation including type-specific rules + List validationMessages = jsonSchema.validate(fieldValue); + if (!validationMessages.isEmpty()) { + throw new IllegalArgumentException( + CatalogExceptionMessage.jsonValidationError(fieldName, validationMessages.toString())); + } + } + } + + public static Object validateAndTransformExtension(Object extension, String entityTypeName) { + if (extension == null) { + return null; + } + + // Validate custom properties existence and schema compliance + validateExtension(extension, entityTypeName); + + // Apply property type-specific transformations (date formatting, enum sorting, etc.) + JsonNode extensionNode = JsonUtils.valueToTree(extension); + if (!extensionNode.isObject()) { + return null; + } + ObjectNode jsonNode = (ObjectNode) extensionNode; + Iterator> customFields = jsonNode.fields(); + + while (customFields.hasNext()) { + Entry entry = customFields.next(); + String fieldName = entry.getKey(); + JsonNode fieldValue = entry.getValue(); + + String customPropertyType = TypeRegistry.getCustomPropertyType(entityTypeName, fieldName); + String propertyConfig = TypeRegistry.getCustomPropertyConfig(entityTypeName, fieldName); + + switch (customPropertyType) { + case "date-cp", "dateTime-cp", "time-cp" -> { + String formattedValue = + getFormattedDateTimeField( + fieldValue.textValue(), customPropertyType, propertyConfig, fieldName); + jsonNode.put(fieldName, formattedValue); + } + case "table-cp" -> validateTableType(fieldValue, propertyConfig, fieldName); + case "enum" -> { + validateEnumKeys(fieldName, fieldValue, propertyConfig); + List enumValues = + StreamSupport.stream(fieldValue.spliterator(), false) + .map(JsonNode::asText) + .sorted() + .collect(Collectors.toList()); + jsonNode.set(fieldName, JsonUtils.valueToTree(enumValues)); + } + case "hyperlink-cp" -> validateHyperlinkUrl(fieldValue, fieldName); + case "entityReference" -> EntityUtil.validateCustomPropertyEntityReference( + fieldValue, fieldName); + case "entityReferenceList" -> EntityUtil.validateCustomPropertyEntityReferenceList( + fieldValue, fieldName); + default -> {} + } + } + + return JsonUtils.treeToValue(jsonNode, Object.class); + } + + private static void validateHyperlinkUrl(JsonNode fieldValue, String fieldName) { + if (fieldValue == null || fieldValue.isNull()) { + return; + } + JsonNode urlNode = fieldValue.get("url"); + if (urlNode == null || urlNode.isNull() || urlNode.asText().isEmpty()) { + return; + } + String url = urlNode.asText(); + try { + java.net.URI uri = new java.net.URI(url); + String scheme = uri.getScheme(); + if (scheme == null + || (!scheme.equalsIgnoreCase("http") && !scheme.equalsIgnoreCase("https"))) { + throw new IllegalArgumentException( + String.format( + "Invalid URL protocol for field '%s': URL must use http or https protocol", + fieldName)); + } + } catch (java.net.URISyntaxException e) { + throw new IllegalArgumentException( + String.format("Invalid URL format for field '%s': %s", fieldName, e.getMessage())); + } + } + + private static String getFormattedDateTimeField( + String fieldValue, String customPropertyType, String propertyConfig, String fieldName) { + DateTimeFormatter formatter; + + try { + return switch (customPropertyType) { + case "date-cp" -> { + DateTimeFormatter inputFormatter = + DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH); + TemporalAccessor date = inputFormatter.parse(fieldValue); + DateTimeFormatter outputFormatter = + DateTimeFormatter.ofPattern(propertyConfig, Locale.ENGLISH); + yield outputFormatter.format(date); + } + case "dateTime-cp" -> { + formatter = DateTimeFormatter.ofPattern(propertyConfig); + LocalDateTime dateTime = LocalDateTime.parse(fieldValue, formatter); + yield dateTime.format(formatter); + } + case "time-cp" -> { + formatter = DateTimeFormatter.ofPattern(propertyConfig); + LocalTime time = LocalTime.parse(fieldValue, formatter); + yield time.format(formatter); + } + default -> throw new IllegalArgumentException( + "Unsupported customPropertyType: " + customPropertyType); + }; + } catch (DateTimeParseException e) { + throw new IllegalArgumentException( + CatalogExceptionMessage.dateTimeValidationError(fieldName, propertyConfig)); + } + } + + private static void validateTableType( + JsonNode fieldValue, String propertyConfig, String fieldName) { + TableConfig tableConfig = + JsonUtils.convertValue(JsonUtils.readTree(propertyConfig), TableConfig.class); + org.openmetadata.schema.type.customProperties.Table tableValue = + JsonUtils.convertValue( + JsonUtils.readTree(String.valueOf(fieldValue)), + org.openmetadata.schema.type.customProperties.Table.class); + Set configColumns = tableConfig.getColumns(); + + try { + JsonUtils.validateJsonSchema( + tableValue, org.openmetadata.schema.type.customProperties.Table.class); + + Set fieldColumns = new HashSet<>(); + fieldValue.get("columns").forEach(column -> fieldColumns.add(column.asText())); + + Set undefinedColumns = new HashSet<>(fieldColumns); + undefinedColumns.removeAll(configColumns); + if (!undefinedColumns.isEmpty()) { + throw new IllegalArgumentException( + "Expected columns: " + + configColumns + + ", but found undefined columns: " + + undefinedColumns); + } + + Set rowFieldNames = new HashSet<>(); + fieldValue.get("rows").forEach(row -> row.fieldNames().forEachRemaining(rowFieldNames::add)); + + undefinedColumns = new HashSet<>(rowFieldNames); + undefinedColumns.removeAll(configColumns); + if (!undefinedColumns.isEmpty()) { + throw new IllegalArgumentException("Rows contain undefined columns: " + undefinedColumns); + } + } catch (ConstraintViolationException e) { + String validationErrors = + e.getConstraintViolations().stream() + .map(violation -> violation.getPropertyPath() + " " + violation.getMessage()) + .collect(Collectors.joining(", ")); + + throw new IllegalArgumentException( + CatalogExceptionMessage.jsonValidationError(fieldName, validationErrors)); + } + } + + public static void validateEnumKeys( + String fieldName, JsonNode fieldValue, String propertyConfig) { + JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig); + EnumConfig config = JsonUtils.treeToValue(propertyConfigNode, EnumConfig.class); + + if (!config.getMultiSelect() && fieldValue.size() > 1) { + throw new IllegalArgumentException( + String.format("Only one value allowed for non-multiSelect %s property", fieldName)); + } + Set validValues = new HashSet<>(config.getValues()); + Set fieldValues = new HashSet<>(); + fieldValue.forEach(value -> fieldValues.add(value.asText())); + + if (!validValues.containsAll(fieldValues)) { + fieldValues.removeAll(validValues); + throw new IllegalArgumentException( + String.format("Values '%s' not supported for property %s", fieldValues, fieldName)); + } + } +} From 89c3e93800ab983f5c9b34ace5e44239117519f2 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 19:07:35 -0700 Subject: [PATCH 08/13] perf(jdbi3): make report_data delete sargable via UTC timestamp range (A9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deleteReportDataTypeAtDate's Postgres form recomputed DATE(TO_TIMESTAMP((json->>'timestamp')::bigint/1000)) per row — non-sargable, a full seq scan across all report types. Replace both engine forms with a single half-open range over the indexed `timestamp` column: WHERE entityFQNHash = :reportDataType AND timestamp >= :startTs AND timestamp < :endTs The caller derives [startTs, endTs) from the yyyy-MM-dd date in UTC via the existing TimestampUtils helpers — matching how the date string and the Elasticsearch cleanup (doc['timestamp'].toLocalDate()) are already built, so semantics are preserved while the query becomes identical and index-backed on both engines. Container-validated on MySQL 8.0 (EXPLAIN type=range) and Postgres 16 (Index Scan using idx_report_data_ts_keyset): exactly the target UTC day is deleted; the preceding-day-23:59, next-day-00:00, and other-entity boundary rows are preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/ReportDataRepository.java | 5 ++++- .../service/jdbi3/TimeSeriesDAOs.java | 22 +++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ReportDataRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ReportDataRepository.java index e2a2251e5cf5..3f9858e96a57 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ReportDataRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ReportDataRepository.java @@ -7,6 +7,7 @@ import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.schema.utils.ResultList; import org.openmetadata.service.Entity; +import org.openmetadata.service.apps.bundles.insights.utils.TimestampUtils; import org.openmetadata.service.resources.analytics.ReportDataResource; public class ReportDataRepository extends EntityTimeSeriesRepository { @@ -34,8 +35,10 @@ public ResultList getReportData( } public void deleteReportDataAtDate(ReportDataType reportDataType, String date) { + long startTs = TimestampUtils.getTimestampFromDateString(date); + long endTs = TimestampUtils.addDays(startTs, 1); ((CollectionDAO.ReportDataTimeSeriesDAO) timeSeriesDao) - .deleteReportDataTypeAtDate(reportDataType.value(), date); + .deleteReportDataTypeAtDate(reportDataType.value(), startTs, endTs); cleanUpIndex(reportDataType, date); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java index 4589b0801e51..25b81cb7d142 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TimeSeriesDAOs.java @@ -1251,16 +1251,20 @@ default String getTimeSeriesTableName() { return "report_data_time_series"; } - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType and date = :date", - connectionType = MYSQL) - @ConnectionAwareSqlUpdate( - value = - "DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType and DATE(TO_TIMESTAMP((json ->> 'timestamp')::bigint/1000)) = DATE(:date)", - connectionType = POSTGRES) + // Half-open UTC millis range [startTs, endTs) over the indexed `timestamp` column. The previous + // Postgres form recomputed DATE(TO_TIMESTAMP(json->>'timestamp'/1000)) per row (non-sargable, + // full + // scan); this seeks idx_report_data_ts_keyset on both engines. Callers derive the bounds from + // the + // yyyy-MM-dd date in UTC (TimestampUtils), matching how the date string + the ES cleanup are + // built. + @SqlUpdate( + "DELETE FROM report_data_time_series " + + "WHERE entityFQNHash = :reportDataType AND timestamp >= :startTs AND timestamp < :endTs") void deleteReportDataTypeAtDate( - @BindFQN("reportDataType") String reportDataType, @Bind("date") String date); + @BindFQN("reportDataType") String reportDataType, + @Bind("startTs") long startTs, + @Bind("endTs") long endTs); @SqlUpdate("DELETE FROM report_data_time_series WHERE entityFQNHash = :reportDataType") void deletePreviousReportData(@BindFQN("reportDataType") String reportDataType); From 3fcf3f4c8f9a47b86b4b801515f58ada3926c45d Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sat, 6 Jun 2026 19:07:45 -0700 Subject: [PATCH 09/13] perf(jdbi3): batch EntityReference resolution in user/team bulk fetchers (B4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bulk field fetchers batched their relationship records but then resolved each EntityReference one row at a time via Entity.getEntityReferenceById — an N+1 over a page of users/teams (teams, roles, personas, default/inherited personas, domains; team users, defaultRoles, defaultPersona, parents, policies). Add a shared EntityRepository.batchResolveRefs(entityType, ids) that resolves a homogeneous id set in one getEntityReferencesByIds call and returns an id->ref map, and route all 13 single-type fetchers through it. The mixed-type owns/follows fetchers keep their per-row resolution: they resolve heterogeneous entity types with defensive per-row try/catch for dangling owned/followed entities, which a batched find() (throws on the first missing id) would regress. Full service suite green (5,892 tests, 0 failures). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/EntityRepository.java | 17 ++++ .../service/jdbi3/TeamRepository.java | 52 +++++++---- .../service/jdbi3/UserRepository.java | 88 ++++++++++++------- 3 files changed, 107 insertions(+), 50 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 9baf5fa1860f..b0a37e9dc370 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1223,6 +1223,23 @@ public final List getReferences(List id, Include include) return find(id, include).stream().map(EntityInterface::getEntityReference).toList(); } + /** + * Batch-resolve EntityReferences for a homogeneous set of ids in one lookup, returning a map keyed + * by id (unresolved ids are absent). Used by bulk field fetchers to avoid per-row {@link + * Entity#getEntityReferenceById} calls. + */ + protected Map batchResolveRefs(String entityType, List ids) { + List distinctIds = ids.stream().distinct().toList(); + Map refsById = new HashMap<>(); + if (!distinctIds.isEmpty()) { + for (EntityReference ref : + Entity.getEntityReferencesByIds(entityType, distinctIds, Include.ALL)) { + refsById.put(ref.getId(), ref); + } + } + return refsById; + } + /** * Find method is used for getting an entity only with core fields stored as JSON without any relational fields set */ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java index 319cc26f7255..a8ff3415a97c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TeamRepository.java @@ -194,13 +194,16 @@ private void fetchAndSetUsers(List teams, Fields fields) { .relationshipDAO() .findToBatch(teamIds, Relationship.HAS.ordinal(), TEAM, Entity.USER); + Map userRefsById = + batchResolveRefs( + Entity.USER, userRecords.stream().map(r -> UUID.fromString(r.getToId())).toList()); Map> teamToUsers = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : userRecords) { UUID teamId = UUID.fromString(record.getFromId()); - EntityReference userRef = - Entity.getEntityReferenceById( - Entity.USER, UUID.fromString(record.getToId()), Include.ALL); - teamToUsers.computeIfAbsent(teamId, k -> new ArrayList<>()).add(userRef); + EntityReference userRef = userRefsById.get(UUID.fromString(record.getToId())); + if (userRef != null) { + teamToUsers.computeIfAbsent(teamId, k -> new ArrayList<>()).add(userRef); + } } for (Team team : teams) { @@ -221,13 +224,16 @@ private void fetchAndSetDefaultRoles(List teams, Fields fields) { .relationshipDAO() .findToBatch(teamIds, Relationship.HAS.ordinal(), TEAM, Entity.ROLE); + Map roleRefsById = + batchResolveRefs( + Entity.ROLE, roleRecords.stream().map(r -> UUID.fromString(r.getToId())).toList()); Map> teamToRoles = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : roleRecords) { UUID teamId = UUID.fromString(record.getFromId()); - EntityReference roleRef = - Entity.getEntityReferenceById( - Entity.ROLE, UUID.fromString(record.getToId()), Include.ALL); - teamToRoles.computeIfAbsent(teamId, k -> new ArrayList<>()).add(roleRef); + EntityReference roleRef = roleRefsById.get(UUID.fromString(record.getToId())); + if (roleRef != null) { + teamToRoles.computeIfAbsent(teamId, k -> new ArrayList<>()).add(roleRef); + } } for (Team team : teams) { @@ -248,13 +254,15 @@ private void fetchAndSetDefaultPersona(List teams, Fields fields) { .relationshipDAO() .findToBatch(teamIds, Relationship.HAS.ordinal(), TEAM, Entity.PERSONA); + Map personaRefsById = + batchResolveRefs( + Entity.PERSONA, + personaRecords.stream().map(r -> UUID.fromString(r.getToId())).toList()); Map teamToPersona = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : personaRecords) { UUID teamId = UUID.fromString(record.getFromId()); - EntityReference personaRef = - Entity.getEntityReferenceById( - Entity.PERSONA, UUID.fromString(record.getToId()), Include.ALL); - if (!Boolean.TRUE.equals(personaRef.getDeleted())) { + EntityReference personaRef = personaRefsById.get(UUID.fromString(record.getToId())); + if (personaRef != null && !Boolean.TRUE.equals(personaRef.getDeleted())) { teamToPersona.put(teamId, personaRef); } } @@ -276,12 +284,16 @@ private void fetchAndSetParents(List teams, Fields fields) { .relationshipDAO() .findFromBatch(teamIds, Relationship.PARENT_OF.ordinal(), TEAM, TEAM); + Map parentRefsById = + batchResolveRefs( + TEAM, parentRecords.stream().map(r -> UUID.fromString(r.getFromId())).toList()); Map> teamToParents = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : parentRecords) { UUID teamId = UUID.fromString(record.getToId()); - EntityReference parentRef = - Entity.getEntityReferenceById(TEAM, UUID.fromString(record.getFromId()), Include.ALL); - teamToParents.computeIfAbsent(teamId, k -> new ArrayList<>()).add(parentRef); + EntityReference parentRef = parentRefsById.get(UUID.fromString(record.getFromId())); + if (parentRef != null) { + teamToParents.computeIfAbsent(teamId, k -> new ArrayList<>()).add(parentRef); + } } for (Team team : teams) { @@ -310,12 +322,16 @@ private void fetchAndSetPolicies(List teams, Fields fields) { .relationshipDAO() .findToBatch(teamIds, Relationship.HAS.ordinal(), TEAM, POLICY); + Map policyRefsById = + batchResolveRefs( + POLICY, policyRecords.stream().map(r -> UUID.fromString(r.getToId())).toList()); Map> teamToPolicies = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : policyRecords) { UUID teamId = UUID.fromString(record.getFromId()); - EntityReference policyRef = - Entity.getEntityReferenceById(POLICY, UUID.fromString(record.getToId()), Include.ALL); - teamToPolicies.computeIfAbsent(teamId, k -> new ArrayList<>()).add(policyRef); + EntityReference policyRef = policyRefsById.get(UUID.fromString(record.getToId())); + if (policyRef != null) { + teamToPolicies.computeIfAbsent(teamId, k -> new ArrayList<>()).add(policyRef); + } } for (Team team : teams) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java index ff12cf1366b2..10371eaeb181 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/UserRepository.java @@ -784,13 +784,16 @@ private void fetchAndSetTeams(List users, Fields fields) { .relationshipDAO() .findFromBatch(userIds, Relationship.HAS.ordinal(), Entity.TEAM, USER); + Map teamRefsById = + batchResolveRefs( + Entity.TEAM, teamRecords.stream().map(r -> UUID.fromString(r.getFromId())).toList()); Map> userToTeams = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : teamRecords) { UUID userId = UUID.fromString(record.getToId()); - EntityReference teamRef = - Entity.getEntityReferenceById( - Entity.TEAM, UUID.fromString(record.getFromId()), Include.ALL); - userToTeams.computeIfAbsent(userId, k -> new ArrayList<>()).add(teamRef); + EntityReference teamRef = teamRefsById.get(UUID.fromString(record.getFromId())); + if (teamRef != null) { + userToTeams.computeIfAbsent(userId, k -> new ArrayList<>()).add(teamRef); + } } for (User user : users) { @@ -826,13 +829,16 @@ private void fetchAndSetRoles(List users, Fields fields) { .relationshipDAO() .findToBatch(userIds, Relationship.HAS.ordinal(), USER, Entity.ROLE); + Map roleRefsById = + batchResolveRefs( + Entity.ROLE, roleRecords.stream().map(r -> UUID.fromString(r.getToId())).toList()); Map> userToRoles = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : roleRecords) { UUID userId = UUID.fromString(record.getFromId()); - EntityReference roleRef = - Entity.getEntityReferenceById( - Entity.ROLE, UUID.fromString(record.getToId()), Include.ALL); - userToRoles.computeIfAbsent(userId, k -> new ArrayList<>()).add(roleRef); + EntityReference roleRef = roleRefsById.get(UUID.fromString(record.getToId())); + if (roleRef != null) { + userToRoles.computeIfAbsent(userId, k -> new ArrayList<>()).add(roleRef); + } } Map> userToTeams = batchFetchTeamsForUsers(userIds); @@ -850,12 +856,13 @@ private Map> batchFetchTeamsForUsers(List us daoCollection .relationshipDAO() .findFromBatch(userIds, Relationship.HAS.ordinal(), Entity.TEAM, Include.ALL); + Map teamRefsById = + batchResolveRefs( + Entity.TEAM, teamRecords.stream().map(r -> UUID.fromString(r.getFromId())).toList()); for (CollectionDAO.EntityRelationshipObject record : teamRecords) { UUID userId = UUID.fromString(record.getToId()); - EntityReference teamRef = - Entity.getEntityReferenceById( - Entity.TEAM, UUID.fromString(record.getFromId()), Include.ALL); - if (!Boolean.TRUE.equals(teamRef.getDeleted())) { + EntityReference teamRef = teamRefsById.get(UUID.fromString(record.getFromId())); + if (teamRef != null && !Boolean.TRUE.equals(teamRef.getDeleted())) { userToTeams.computeIfAbsent(userId, k -> new ArrayList<>()).add(teamRef); } } @@ -896,12 +903,15 @@ private void fetchAndSetOwns(List users, Fields fields) { daoCollection .relationshipDAO() .findFromBatch(userIds, Relationship.HAS.ordinal(), Entity.TEAM, USER); + Map teamRefsById = + batchResolveRefs( + Entity.TEAM, teamRecords.stream().map(r -> UUID.fromString(r.getFromId())).toList()); for (CollectionDAO.EntityRelationshipObject record : teamRecords) { UUID userId = UUID.fromString(record.getToId()); - EntityReference teamRef = - Entity.getEntityReferenceById( - Entity.TEAM, UUID.fromString(record.getFromId()), Include.ALL); - userTeams.computeIfAbsent(userId, k -> new ArrayList<>()).add(teamRef); + EntityReference teamRef = teamRefsById.get(UUID.fromString(record.getFromId())); + if (teamRef != null) { + userTeams.computeIfAbsent(userId, k -> new ArrayList<>()).add(teamRef); + } } } else { // Use already fetched teams @@ -1019,13 +1029,17 @@ private void fetchAndSetPersonas(List users, Fields fields) { .relationshipDAO() .findFromBatch(userIds, Relationship.APPLIED_TO.ordinal(), Entity.PERSONA, USER); + Map personaRefsById = + batchResolveRefs( + Entity.PERSONA, + personaRecords.stream().map(r -> UUID.fromString(r.getFromId())).toList()); Map> userToPersonas = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : personaRecords) { UUID userId = UUID.fromString(record.getToId()); - EntityReference personaRef = - Entity.getEntityReferenceById( - Entity.PERSONA, UUID.fromString(record.getFromId()), Include.ALL); - userToPersonas.computeIfAbsent(userId, k -> new ArrayList<>()).add(personaRef); + EntityReference personaRef = personaRefsById.get(UUID.fromString(record.getFromId())); + if (personaRef != null) { + userToPersonas.computeIfAbsent(userId, k -> new ArrayList<>()).add(personaRef); + } } for (User user : users) { @@ -1046,13 +1060,17 @@ private void fetchAndSetDefaultPersona(List users, Fields fields) { .relationshipDAO() .findFromBatch(userIds, Relationship.DEFAULTS_TO.ordinal(), Entity.PERSONA, USER); + Map defaultPersonaRefsById = + batchResolveRefs( + Entity.PERSONA, + defaultPersonaRecords.stream().map(r -> UUID.fromString(r.getFromId())).toList()); Map userToDefaultPersona = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : defaultPersonaRecords) { UUID userId = UUID.fromString(record.getToId()); - EntityReference personaRef = - Entity.getEntityReferenceById( - Entity.PERSONA, UUID.fromString(record.getFromId()), Include.ALL); - userToDefaultPersona.put(userId, personaRef); + EntityReference personaRef = defaultPersonaRefsById.get(UUID.fromString(record.getFromId())); + if (personaRef != null) { + userToDefaultPersona.put(userId, personaRef); + } } for (User user : users) { @@ -1110,13 +1128,15 @@ private void fetchAndSetInheritedPersonas(List users, Fields fields) { .findToBatch( new ArrayList<>(allTeamIds), Relationship.HAS.ordinal(), TEAM, Entity.PERSONA); + Map inheritedPersonaRefsById = + batchResolveRefs( + Entity.PERSONA, + personaRecords.stream().map(r -> UUID.fromString(r.getToId())).toList()); Map teamToPersona = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : personaRecords) { UUID teamId = UUID.fromString(record.getFromId()); - EntityReference personaRef = - Entity.getEntityReferenceById( - Entity.PERSONA, UUID.fromString(record.getToId()), Include.ALL); - if (!Boolean.TRUE.equals(personaRef.getDeleted())) { + EntityReference personaRef = inheritedPersonaRefsById.get(UUID.fromString(record.getToId())); + if (personaRef != null && !Boolean.TRUE.equals(personaRef.getDeleted())) { teamToPersona.put(teamId, personaRef); } } @@ -1152,13 +1172,17 @@ private void fetchAndSetDomains(List users, Fields fields) { .relationshipDAO() .findFromBatch(userIds, Relationship.HAS.ordinal(), Entity.DOMAIN, USER); + Map domainRefsById = + batchResolveRefs( + Entity.DOMAIN, + domainRecords.stream().map(r -> UUID.fromString(r.getFromId())).toList()); Map> userToDomains = new HashMap<>(); for (CollectionDAO.EntityRelationshipObject record : domainRecords) { UUID userId = UUID.fromString(record.getToId()); - EntityReference domainRef = - Entity.getEntityReferenceById( - Entity.DOMAIN, UUID.fromString(record.getFromId()), Include.ALL); - userToDomains.computeIfAbsent(userId, k -> new ArrayList<>()).add(domainRef); + EntityReference domainRef = domainRefsById.get(UUID.fromString(record.getFromId())); + if (domainRef != null) { + userToDomains.computeIfAbsent(userId, k -> new ArrayList<>()).add(domainRef); + } } for (User user : users) { From f0ef39669142594a539343811a5f98c5548e3c98 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sun, 7 Jun 2026 09:05:33 -0700 Subject: [PATCH 10/13] =?UTF-8?q?test+docs:=20address=20PR=20#28778=20revi?= =?UTF-8?q?ew=20=E2=80=94=20exhaustive=20composition=20guard=20+=20isBot?= =?UTF-8?q?=20migration=20lock=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CollectionDAOCompositionTest no longer hardcodes a ~35-entry accessor map; it now derives the expected set by reflecting over CollectionDAO and every interface it transitively extends, asserting each @CreateSqlObject accessor stays visible via getMethods() and keeps its annotation through CollectionDAO. A MIN_EXPECTED_ACCESSORS floor fails loudly if the reflection scan ever returns nothing. The guard now stays exhaustive automatically as accessors/aggregator interfaces evolve (review: a non-listed accessor losing visibility would previously go uncaught). - Document the operational cost of the Postgres isBot fix: ADD COLUMN ... STORED rewrites user_entity under an ACCESS EXCLUSIVE lock and the backfill UPDATE scans the full table; added a maintenance-window heads-up to the migration (review: large-table lock stall). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../native/2.0.0/postgres/schemaChanges.sql | 5 + .../jdbi3/CollectionDAOCompositionTest.java | 103 +++++++++--------- 2 files changed, 58 insertions(+), 50 deletions(-) diff --git a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql index 33ba39a50f78..43576c1876f2 100644 --- a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql @@ -344,6 +344,11 @@ CREATE INDEX IF NOT EXISTS idx_entity_usage_entitytype_usagedate -- isBot column filter) was therefore wrong on Postgres. Postgres cannot alter a generated -- column's expression in place, so backfill any rows missing $.isBot, drop the column -- (this also drops idx_isBot) and recreate it reading the correct path. +-- Operational note: ADD COLUMN ... STORED rewrites the whole user_entity table and holds an +-- ACCESS EXCLUSIVE lock for its duration, and the backfill UPDATE below also scans the full +-- table. On deployments with a very large user_entity, run this migration in a maintenance +-- window; runtime scales with row count (typically seconds, but minutes for millions of users). +-- The change is one-time, idempotent, and Postgres-only (MySQL 1.6.3 was already correct). UPDATE user_entity SET json = jsonb_set(json, '{isBot}', 'false'::jsonb, true) WHERE (json ->> 'isBot') IS NULL; ALTER TABLE user_entity DROP COLUMN IF EXISTS isBot; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/CollectionDAOCompositionTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/CollectionDAOCompositionTest.java index 616d69c333b7..5f3443d64726 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/CollectionDAOCompositionTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/CollectionDAOCompositionTest.java @@ -16,8 +16,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Map; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; import org.jdbi.v3.sqlobject.CreateSqlObject; @@ -25,72 +27,73 @@ /** * Guards the CollectionDAO decomposition. CollectionDAO was split into domain aggregator - * interfaces (RdfInfraDAOs, SearchReindexDAOs, ...) that it `extends`. JDBI's SqlObjectFactory + * interfaces (RdfInfraDAOs, SearchReindexDAOs, ...) that it {@code extends}. JDBI's SqlObjectFactory * builds its handler map from {@code sqlObjectType.getMethods()}, which returns inherited public * methods, so every {@code @CreateSqlObject} accessor moved to an extended interface must remain * visible (and annotated) through CollectionDAO for the runtime wiring to keep working. + * + *

The expected accessor set is derived by reflection over CollectionDAO and every interface it + * transitively extends, so the guard stays exhaustive automatically as accessors are added, moved, + * or new aggregator interfaces appear — no hardcoded list to keep in sync. */ class CollectionDAOCompositionTest { - private static final Map MOVED_ACCESSOR_TO_INTERFACE = - Map.ofEntries( - Map.entry("rdfIndexJobDAO", "RdfInfraDAOs"), - Map.entry("rdfIndexServerStatsDAO", "RdfInfraDAOs"), - Map.entry("searchIndexJobDAO", "SearchReindexDAOs"), - Map.entry("searchIndexServerStatsDAO", "SearchReindexDAOs"), - Map.entry("aiApplicationDAO", "AiGovernanceDAOs"), - Map.entry("mcpServiceDAO", "AiGovernanceDAOs"), - Map.entry("feedDAO", "FeedDAOs"), - Map.entry("taskDAO", "FeedDAOs"), - Map.entry("tagUsageDAO", "ClassificationTagDAOs"), - Map.entry("classificationDAO", "ClassificationTagDAOs"), - Map.entry("testCaseDAO", "TimeSeriesDAOs"), - Map.entry("testCaseResultTimeSeriesDao", "TimeSeriesDAOs"), - Map.entry("activityStreamDAO", "ActivityAuditDAOs"), - Map.entry("auditLogDAO", "ActivityAuditDAOs"), - Map.entry("domainDAO", "GovernanceDAOs"), - Map.entry("dataContractDAO", "GovernanceDAOs"), - Map.entry("eventSubscriptionDAO", "EventSubscriptionDAOs"), - Map.entry("folderDAO", "KnowledgeAssetDAOs"), - Map.entry("knowledgePageDAO", "KnowledgeAssetDAOs"), - Map.entry("systemDAO", "SystemTokenDAOs"), - Map.entry("getTokenDAO", "SystemTokenDAOs"), - Map.entry("databaseDAO", "DataAssetServiceDAOs"), - Map.entry("dashboardDAO", "DataAssetServiceDAOs"), - Map.entry("containerDAO", "DataAssetServiceDAOs"), - Map.entry("dbServiceDAO", "DataAssetServiceDAOs"), - Map.entry("tableDAO", "EntityDataDAOs"), - Map.entry("pipelineDAO", "EntityDataDAOs"), - Map.entry("userDAO", "AccessControlDAOs"), - Map.entry("teamDAO", "AccessControlDAOs"), - Map.entry("changeEventDAO", "AccessControlDAOs"), - Map.entry("workflowDAO", "WorkflowDocStoreDAOs"), - Map.entry("oauthClientDAO", "OAuthDAOs"), - Map.entry("relationshipDAO", "CoreRelationshipDAOs"), - Map.entry("fieldRelationshipDAO", "CoreRelationshipDAOs"), - Map.entry("entityExtensionDAO", "CoreRelationshipDAOs")); + /** A floor so a reflection change that silently returns nothing fails loudly. */ + private static final int MIN_EXPECTED_ACCESSORS = 30; @Test - void inheritedCreateSqlObjectAccessorsRemainVisibleAndAnnotated() throws NoSuchMethodException { + void everyCreateSqlObjectAccessorRemainsVisibleAndAnnotated() throws NoSuchMethodException { Set visibleMethods = Arrays.stream(CollectionDAO.class.getMethods()) .map(Method::getName) .collect(Collectors.toSet()); - for (Map.Entry entry : MOVED_ACCESSOR_TO_INTERFACE.entrySet()) { - String accessor = entry.getKey(); + List accessors = declaredCreateSqlObjectAccessors(); + assertTrue( + accessors.size() >= MIN_EXPECTED_ACCESSORS, + "Expected at least " + + MIN_EXPECTED_ACCESSORS + + " @CreateSqlObject accessors across CollectionDAO and its extended interfaces, found " + + accessors.size()); + + for (Method accessor : accessors) { + String name = accessor.getName(); + String declaredIn = accessor.getDeclaringClass().getSimpleName(); assertTrue( - visibleMethods.contains(accessor), - accessor + " (moved to " + entry.getValue() + ") is not visible via getMethods()"); - Method method = CollectionDAO.class.getMethod(accessor); + visibleMethods.contains(name), + name + " (declared on " + declaredIn + ") is not visible via CollectionDAO.getMethods()"); + Method viaCollectionDao = CollectionDAO.class.getMethod(name); assertNotNull( - method.getAnnotation(CreateSqlObject.class), - accessor + " lost its @CreateSqlObject annotation after the move"); + viaCollectionDao.getAnnotation(CreateSqlObject.class), + name + " lost its @CreateSqlObject annotation when resolved through CollectionDAO"); } } - @Test - void accessorsDeclaredDirectlyOnCollectionDaoStillWire() throws NoSuchMethodException { - assertNotNull(CollectionDAO.class.getMethod("assetDAO").getAnnotation(CreateSqlObject.class)); + /** + * Every {@code @CreateSqlObject} accessor declared directly on CollectionDAO or on any interface + * it transitively extends — the full source-of-truth set JDBI must be able to wire. + */ + private static List declaredCreateSqlObjectAccessors() { + Set> types = new LinkedHashSet<>(); + types.add(CollectionDAO.class); + collectExtendedInterfaces(CollectionDAO.class, types); + + List accessors = new ArrayList<>(); + for (Class type : types) { + for (Method method : type.getDeclaredMethods()) { + if (method.isAnnotationPresent(CreateSqlObject.class)) { + accessors.add(method); + } + } + } + return accessors; + } + + private static void collectExtendedInterfaces(Class type, Set> accumulator) { + for (Class extended : type.getInterfaces()) { + if (accumulator.add(extended)) { + collectExtendedInterfaces(extended, accumulator); + } + } } } From 4b8cc2132040782f8cf0bd50d5d824255c6e2a7e Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sun, 7 Jun 2026 14:56:39 -0700 Subject: [PATCH 11/13] =?UTF-8?q?Revert=20"perf(jdbi3):=20data=5Fcontract?= =?UTF-8?q?=20generated=20columns=20+=20index=20(A3)"=20=E2=80=94=20broke?= =?UTF-8?q?=20fresh=20installs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI integration tests (MySQL: 220 errors, Python ometa: 8 failures) failed with "Unknown column 'contractEntityId'" from DataContractDAO.getContractByEntityId. The A3 optimization rewrote that query to read VIRTUAL/STORED generated columns added by the 2.0.0 migration, but the columns are absent in the fresh-install / test-provisioned schema (the MySQL guarded ADD COLUMN did not materialize them), so every getContractByEntityId call — including BaseEntityIT.get_entityDataContract_200 run for every entity type — threw a SQL syntax error. Restore the original JSON-path query (JSON_EXTRACT / json#>>'{...}') that works on any schema, and drop the generated-column + index migration blocks from both 2.0.0 schemaChanges.sql files. data_contract_entity is a small write-path table, so the lost index seek is negligible; correctness on fresh installs matters more. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../native/2.0.0/mysql/schemaChanges.sql | 53 ------------------- .../native/2.0.0/postgres/schemaChanges.sql | 12 ----- .../service/jdbi3/GovernanceDAOs.java | 4 +- 3 files changed, 2 insertions(+), 67 deletions(-) diff --git a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql index bb85de91f52d..77ca00635852 100644 --- a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql @@ -400,56 +400,3 @@ SET @ddl = ( PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; - --- Perf (A3): getContractByEntityId filtered $.entity.id / $.entity.type via JSON_EXTRACT --- (nested, unindexed paths) on the per-entity contract-enforcement write path, full-scanning --- data_contract_entity. Add VIRTUAL generated columns mirroring those paths plus a composite --- index so the lookup becomes an index seek. VIRTUAL columns need no table rewrite. MySQL has --- no ADD COLUMN/KEY IF NOT EXISTS, so guard each add via information_schema. -SET @ddl = ( - SELECT IF( - EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = DATABASE() - AND table_name = 'data_contract_entity' - AND column_name = 'contractEntityId' - ), - 'SELECT 1', - 'ALTER TABLE data_contract_entity ADD COLUMN contractEntityId VARCHAR(36) GENERATED ALWAYS AS (json ->> ''$.entity.id'') VIRTUAL' - ) -); -PREPARE stmt FROM @ddl; -EXECUTE stmt; -DEALLOCATE PREPARE stmt; - -SET @ddl = ( - SELECT IF( - EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_schema = DATABASE() - AND table_name = 'data_contract_entity' - AND column_name = 'contractEntityType' - ), - 'SELECT 1', - 'ALTER TABLE data_contract_entity ADD COLUMN contractEntityType VARCHAR(256) GENERATED ALWAYS AS (json ->> ''$.entity.type'') VIRTUAL' - ) -); -PREPARE stmt FROM @ddl; -EXECUTE stmt; -DEALLOCATE PREPARE stmt; - -SET @ddl = ( - SELECT IF( - EXISTS ( - SELECT 1 FROM information_schema.statistics - WHERE table_schema = DATABASE() - AND table_name = 'data_contract_entity' - AND index_name = 'idx_data_contract_entity_contract_entity' - ), - 'SELECT 1', - 'ALTER TABLE data_contract_entity ADD KEY idx_data_contract_entity_contract_entity (contractEntityId, contractEntityType)' - ) -); -PREPARE stmt FROM @ddl; -EXECUTE stmt; -DEALLOCATE PREPARE stmt; diff --git a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql index 43576c1876f2..7e2bd90f7dc4 100644 --- a/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/postgres/schemaChanges.sql @@ -355,15 +355,3 @@ ALTER TABLE user_entity DROP COLUMN IF EXISTS isBot; ALTER TABLE user_entity ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS ((json ->> 'isBot')::boolean) STORED NOT NULL; CREATE INDEX IF NOT EXISTS idx_isBot ON user_entity (isBot); - --- Perf (A3): getContractByEntityId filtered json#>>'{entity,id}' / '{entity,type}' (nested, --- unindexed paths) on the per-entity contract-enforcement write path, full-scanning --- data_contract_entity. Add STORED generated columns mirroring those paths plus a composite --- index so the lookup becomes an index seek. The #>> operator on jsonb is immutable, so it is --- valid in a STORED generated column; STORED backfills existing rows automatically. -ALTER TABLE data_contract_entity - ADD COLUMN IF NOT EXISTS contractEntityId VARCHAR(36) GENERATED ALWAYS AS (json #>> '{entity,id}') STORED; -ALTER TABLE data_contract_entity - ADD COLUMN IF NOT EXISTS contractEntityType VARCHAR(256) GENERATED ALWAYS AS (json #>> '{entity,type}') STORED; -CREATE INDEX IF NOT EXISTS idx_data_contract_entity_contract_entity - ON data_contract_entity (contractEntityId, contractEntityType); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java index da64cee8e48b..c61bb2891d1b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GovernanceDAOs.java @@ -202,11 +202,11 @@ default String getNameHashColumn() { @ConnectionAwareSqlQuery( value = - "SELECT json FROM data_contract_entity WHERE contractEntityId = :entityId AND contractEntityType = :entityType AND deleted = FALSE LIMIT 1", + "SELECT json FROM data_contract_entity WHERE JSON_EXTRACT(json, '$.entity.id') = :entityId AND JSON_EXTRACT(json, '$.entity.type') = :entityType AND (JSON_EXTRACT(json, '$.deleted') IS NULL OR JSON_EXTRACT(json, '$.deleted') = false) LIMIT 1", connectionType = MYSQL) @ConnectionAwareSqlQuery( value = - "SELECT json FROM data_contract_entity WHERE contractEntityId = :entityId AND contractEntityType = :entityType AND deleted = FALSE LIMIT 1", + "SELECT json FROM data_contract_entity WHERE json#>>'{entity,id}' = :entityId AND json#>>'{entity,type}' = :entityType AND (json->>'deleted' IS NULL OR json->>'deleted' = 'false') LIMIT 1", connectionType = POSTGRES) String getContractByEntityId( @Bind("entityId") String entityId, @Bind("entityType") String entityType); From 2163e858d378fd99b9d3b7c3240c3bbda629acef Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Sun, 7 Jun 2026 15:13:24 -0700 Subject: [PATCH 12/13] fix(lineage): emit ENTITY_LINEAGE_DELETED for source-based bulk deletions deleteLineage and deleteLineageByFQN emit an ENTITY_LINEAGE_DELETED change event, but deleteLineageBySource (the DELETE .../lineage/{entityType}/{entityId}/{lineageSource} endpoint, used to drop all pipeline/OpenLineage edges into an entity) deleted the relations without emitting any event, so consumers/alerts subscribed to lineage-deleted events silently missed source-based bulk deletions. Emit ENTITY_LINEAGE_DELETED for every removed relation (reusing the already-loaded relations and resolveRefForCacheInvalidation), threading the authenticated principal through as deletedBy from the resource endpoint. Addresses PR #28778 review finding #2. Service main+test compile; LineageRepositoryTest (18) passes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/LineageRepository.java | 16 +++++++++++++++- .../resources/lineage/LineageResource.java | 3 ++- .../service/jdbi3/LineageRepositoryTest.java | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java index 1e31dde8e2f5..5a666cf4cfad 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/LineageRepository.java @@ -1091,7 +1091,7 @@ public boolean deleteLineageByFQN( } @Transaction - public void deleteLineageBySource(UUID toId, String toEntity, String source) { + public void deleteLineageBySource(UUID toId, String toEntity, String source, String deletedBy) { List relations; if (source.equals(LineageDetails.Source.PIPELINE_LINEAGE.value()) || source.equals(LineageDetails.Source.OPEN_LINEAGE.value())) { @@ -1110,6 +1110,20 @@ public void deleteLineageBySource(UUID toId, String toEntity, String source) { .deleteLineageBySource(toId, toEntity, source, Relationship.UPSTREAM.ordinal()); } deleteLineageFromSearch(relations); + emitLineageDeletedEvents(relations, deletedBy); + } + + private void emitLineageDeletedEvents( + List relations, String deletedBy) { + for (CollectionDAO.EntityRelationshipObject relation : relations) { + LineageDetails lineageDetails = JsonUtils.readValue(relation.getJson(), LineageDetails.class); + emitLineageChangeEvent( + EventType.ENTITY_LINEAGE_DELETED, + resolveRefForCacheInvalidation(relation.getFromEntity(), relation.getFromId()), + resolveRefForCacheInvalidation(relation.getToEntity(), relation.getToId()), + lineageDetails, + deletedBy); + } } @Transaction diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java index ded19c99263c..b6cb56f7af9e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java @@ -1096,7 +1096,8 @@ public Response deleteLineageByType( securityContext, new OperationContext(LINEAGE_FIELD, MetadataOperation.EDIT_LINEAGE), new LineageResourceContext()); - dao.deleteLineageBySource(entityId, entityType, lineageSource); + dao.deleteLineageBySource( + entityId, entityType, lineageSource, securityContext.getUserPrincipal().getName()); return Response.status(Status.OK).build(); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/LineageRepositoryTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/LineageRepositoryTest.java index b2d846f590f2..250908544ea6 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/LineageRepositoryTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/LineageRepositoryTest.java @@ -834,7 +834,7 @@ void testDeleteLineageBySource_OpenLineage_UsesPipelinePath() { LineageRepository lineageRepository = new LineageRepository(); UUID entityId = UUID.randomUUID(); lineageRepository.deleteLineageBySource( - entityId, "table", LineageDetails.Source.OPEN_LINEAGE.value()); + entityId, "table", LineageDetails.Source.OPEN_LINEAGE.value(), "admin"); org.mockito.Mockito.verify(relationshipDAO) .findLineageBySourcePipeline( From 39bf85f01062bd3ea29ba0f1fc99618a5b83d186 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Tue, 9 Jun 2026 10:46:08 -0700 Subject: [PATCH 13/13] migration: use plain CREATE INDEX for entity_usage(entityType, usageDate) (P2) Replace the information_schema-guarded PREPARE/EXECUTE dynamic-SQL block for the entity_usage composite index with a plain CREATE INDEX, matching the established convention across the native MySQL migrations (the migration framework runs each versioned script exactly once, so the idempotency guard is unnecessary). Postgres already used the standard CREATE INDEX IF NOT EXISTS. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../native/2.0.0/mysql/schemaChanges.sql | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql index 77ca00635852..189107bb86a6 100644 --- a/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/2.0.0/mysql/schemaChanges.sql @@ -382,21 +382,5 @@ CREATE TABLE IF NOT EXISTS `user_session` ( -- filter entity_usage on (entityType, usageDate). The only existing index is -- UNIQUE (id, usageDate), which is unusable for that predicate, so every run full-scans -- the table once per subquery. A composite (entityType, usageDate) index turns the --- percentile subqueries into range scans. MySQL has no `ADD KEY IF NOT EXISTS`, so guard --- via information_schema. -SET @ddl = ( - SELECT IF( - EXISTS ( - SELECT 1 - FROM information_schema.statistics - WHERE table_schema = DATABASE() - AND table_name = 'entity_usage' - AND index_name = 'idx_entity_usage_entitytype_usagedate' - ), - 'SELECT 1', - 'ALTER TABLE entity_usage ADD KEY idx_entity_usage_entitytype_usagedate (entityType, usageDate)' - ) -); -PREPARE stmt FROM @ddl; -EXECUTE stmt; -DEALLOCATE PREPARE stmt; +-- percentile subqueries into range scans. +CREATE INDEX idx_entity_usage_entitytype_usagedate ON entity_usage (entityType, usageDate);