fix: add iam policies and lakeformation changes for legacy loading

infra/orcavault-run/environments/dev/main.tf

@@ -56,6 +56,19 @@ module "orcavault_run" {
   redshift_workgroup_name = "orcahouse-${local.environment}"
   orcavault_release_ref   = var.orcavault_release_ref
   schedule_enabled        = var.schedule_enabled
+
+  glue_databases = [
+    "orcabus_filemanager",
+    "orcabus_metadata_manager",
+    "orcabus_sequence_run_manager",
+    "orcabus_workflow_manager",
+    "data_portal",
+  ]
+  bucket_name = "orcahouse-prod-landing-zone-115253169271-ap-southeast-2-an"
+  s3_prefix = [
+    "orcabus/v1/cdc/",
+    "data-portal/orcavault-legacy-20260623/orcavault/",
+  ]
 }
 
 output "orcavault_run" {

infra/orcavault-run/environments/prod/main.tf

@@ -56,6 +56,19 @@ module "orcavault_run" {
   redshift_workgroup_name = "orcahouse-${local.environment}"
   orcavault_release_ref   = var.orcavault_release_ref
   schedule_enabled        = var.schedule_enabled
+
+  glue_databases = [
+    "orcabus_filemanager",
+    "orcabus_metadata_manager",
+    "orcabus_sequence_run_manager",
+    "orcabus_workflow_manager",
+    "data_portal",
+  ]
+  bucket_name = "orcahouse-prod-landing-zone-115253169271-ap-southeast-2-an"
+  s3_prefix = [
+    "orcabus/v1/cdc/",
+    "data-portal/orcavault-legacy-20260623/orcavault/",
+  ]
 }
 
 output "orcavault_run" {

infra/orcavault-run/modules/orcavault-run/iam.tf

@@ -83,15 +83,15 @@ data "aws_iam_policy_document" "dbt_task" {
     condition {
       test     = "StringLike"
       variable = "s3:prefix"
-      values   = ["${var.cdc_s3_prefix}*"]
+      values   = [for prefix in var.s3_prefix : "${prefix}*"]
     }
   }
 
   statement {
     sid       = "ReadCdcFiles"
     effect    = "Allow"
     actions   = ["s3:GetObject"]
-    resources = ["${data.aws_s3_bucket.cdc.arn}/${var.cdc_s3_prefix}*"]
+    resources = [for prefix in var.s3_prefix : "${data.aws_s3_bucket.cdc.arn}/${prefix}*"]
   }
 
   statement {

infra/orcavault-run/modules/orcavault-run/lakeformation.tf

@@ -1,7 +1,7 @@
 # The Lake Formation grants for the dbt task role.
 
 resource "aws_lakeformation_permissions" "cdc_database" {
-  for_each  = toset(var.cdc_glue_databases)
+  for_each  = toset(var.glue_databases)
   principal = aws_iam_role.dbt_task.arn
 
   database {
@@ -11,7 +11,7 @@ resource "aws_lakeformation_permissions" "cdc_database" {
 }
 
 resource "aws_lakeformation_permissions" "cdc_tables" {
-  for_each  = toset(var.cdc_glue_databases)
+  for_each  = toset(var.glue_databases)
   principal = aws_iam_role.dbt_task.arn
 
   table {

infra/orcavault-run/modules/orcavault-run/main.tf

@@ -65,7 +65,7 @@ data "aws_redshiftserverless_workgroup" "this" {
 }
 
 data "aws_s3_bucket" "cdc" {
-  bucket = var.cdc_bucket_name
+  bucket = var.bucket_name
 }
 
 data "aws_ecr_repository" "dbt" {

infra/orcavault-run/modules/orcavault-run/test_orcavault_run.tftest.hcl

@@ -54,6 +54,9 @@ variables {
   environment             = "prod"
   redshift_workgroup_name = "orcahouse-prod"
   orcavault_release_ref   = "v1.0.0"
+  glue_databases          = ["orcabus_filemanager", "data_portal"]
+  bucket_name             = "landing-zone"
+  s3_prefix               = ["orcabus/v1/cdc/", "data-portal/export/"]
 }
 
 run "schedule_builds_everything" {
@@ -116,6 +119,33 @@ run "task_writes_its_event" {
   }
 }
 
+run "task_reads_source" {
+  assert {
+    condition     = toset(keys(aws_lakeformation_permissions.cdc_tables)) == toset(var.glue_databases)
+    error_message = "the task should be granted the tables of all database"
+  }
+
+  assert {
+    condition = alltrue([
+      for prefix in var.s3_prefix : anytrue([
+        for statement in data.aws_iam_policy_document.dbt_task.statement :
+        try(contains(statement.resources, "${data.aws_s3_bucket.cdc.arn}/${prefix}*"), false)
+      ])
+    ])
+    error_message = "the task should be able to read the files"
+  }
+
+  assert {
+    condition = alltrue([
+      for prefix in var.s3_prefix : anytrue([
+        for statement in data.aws_iam_policy_document.dbt_task.statement :
+        try(contains(one(statement.condition).values, "${prefix}*"), false)
+      ])
+    ])
+    error_message = "the task should be able to list the files under all prefixes"
+  }
+}
+
 run "environments_are_separate" {
   variables {
     environment = "dev"

infra/orcavault-run/modules/orcavault-run/variables.tf

@@ -16,27 +16,19 @@ variable "ecr_repository_name" {
   default     = "orcavault-run-dbt"
 }
 
-variable "cdc_glue_databases" {
-  description = "The Glue databases created by infra/glue-crawler/orcabus-db."
+variable "glue_databases" {
+  description = "The Glue databases the task reads, i.e. the CDC databases created by infra/glue-crawler/orcabus-db, and the legacy data_portal database."
   type        = list(string)
-  default = [
-    "orcabus_filemanager",
-    "orcabus_metadata_manager",
-    "orcabus_sequence_run_manager",
-    "orcabus_workflow_manager",
-  ]
 }
 
-variable "cdc_bucket_name" {
-  description = "The Bucket DMS writes CDC files to, created by infra/dms/orcabus-db."
+variable "bucket_name" {
+  description = "The bucket with the files of the glue_databases tables, i.e. the landing zone that DMS writes CDC files to, created by infra/dms/orcabus-db."
   type        = string
-  default     = "orcahouse-prod-landing-zone-115253169271-ap-southeast-2-an"
 }
 
-variable "cdc_s3_prefix" {
-  description = "The key prefix of the CDC files in cdc_bucket_name."
-  type        = string
-  default     = "orcabus/v1/cdc/"
+variable "s3_prefix" {
+  description = "The key prefixes of the glue_database table files in bucket_name."
+  type        = list(string)
 }
 
 variable "orcavault_release_ref" {
@@ -53,13 +45,13 @@ variable "dbt_threads" {
 variable "task_cpu" {
   description = "The Fargate task CPUs"
   type        = string
-  default     = "512"
+  default     = "4096"
 }
 
 variable "task_memory" {
   description = "The Fargate task memory in MiB."
   type        = string
-  default     = "2048"
+  default     = "8192"
 }
 
 variable "schedule_expression" {

infra/redshift/environments/dev/main.tf

@@ -12,7 +12,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "6.45.0"
+      version = ">= 6.70.0"
     }
   }
 }
@@ -77,7 +77,7 @@ module "redshift_serverless" {
   #
   # See variable description at infra/redshift/modules/redshift-serverless/variables.tf
   base_capacity            = 4   # RPUs
-  max_capacity             = 4   # RPUs
+  max_capacity             = 8   # RPUs
   max_query_execution_time = 900 # in seconds, 15 minutes
   compute_limit_amount     = 120 # Maximum RPU-hours, Monthly
   data_limit_amount        = 1   # TB cross region transfer, Monthly

infra/redshift/environments/prod/main.tf

@@ -12,7 +12,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "6.45.0"
+      version = ">= 6.70.0"
     }
   }
 }
@@ -78,7 +78,7 @@ module "redshift_serverless" {
 
   # See note in infra/redshift/environments/dev/main.tf for tuning
   base_capacity            = 4   # RPUs
-  max_capacity             = 4   # RPUs
+  max_capacity             = 8   # RPUs
   max_query_execution_time = 900 # in seconds, 15 minutes
   compute_limit_amount     = 120 # Maximum RPU-hours, Monthly
   data_limit_amount        = 1   # TB cross region transfer, Monthly