Merge CDA feed handler with ingestion helper

pipeline/terraform/main.tf

@@ -215,37 +215,6 @@ resource "google_cloud_run_v2_service" "ingestion_helper" {
   depends_on = [google_project_service.services]
 }
 
-resource "google_cloud_run_v2_service" "import_helper" {
-  name     = "import-helper-service"
-  location = var.region
-  project  = var.project_id
-
-  template {
-    service_account = google_service_account.automation_sa.email
-    containers {
-      image = "${var.artifact_registry_url}/datacommons-import-helper:latest"
-      env {
-        name  = "PROJECT_ID"
-        value = var.project_id
-      }
-      env {
-        name  = "LOCATION"
-        value = var.region
-      }
-      env {
-        name  = "GCS_BUCKET_ID"
-        value = google_storage_bucket.import_bucket.name
-      }
-      env {
-        name  = "INGESTION_HELPER_URL"
-        value = google_cloud_run_v2_service.ingestion_helper.uri
-      }
-    }
-  }
-
-  depends_on = [google_project_service.services]
-}
-
 # --- Cloud Workflows ---
 
 resource "google_workflows_workflow" "import_automation_workflow" {
@@ -363,7 +332,7 @@ resource "google_pubsub_subscription" "import_automation_sub" {
   filter = "attributes.transfer_status=\"TRANSFER_COMPLETED\""
 
   push_config {
-    push_endpoint = google_cloud_run_v2_service.import_helper.uri
+    push_endpoint = google_cloud_run_v2_service.ingestion_helper.uri
     oidc_token {
       service_account_email = google_service_account.automation_sa.email
     }

pipeline/workflow/cloudbuild.yaml

@@ -41,11 +41,6 @@ steps:
   args: ['builds', 'submit', 'ingestion-helper', '--config', 'ingestion-helper/cloudbuild.yaml', '--substitutions', '_AR_REPO_URL=${_AR_REPO_URL},_VERSION=${_VERSION}']
   dir: 'pipeline/workflow'
 
-- id: 'build-import-helper'
-  name: 'gcr.io/cloud-builders/gcloud'
-  args: ['builds', 'submit', 'import-helper', '--config', 'import-helper/cloudbuild.yaml', '--substitutions', '_AR_REPO_URL=${_AR_REPO_URL},_VERSION=${_VERSION}']
-  dir: 'pipeline/workflow'
-
 - id: 'build-dataflow-template'
   name: 'gcr.io/cloud-builders/gcloud'
   args:

pipeline/workflow/deploy-services.yaml

@@ -37,10 +37,6 @@ steps:
   name: 'gcr.io/cloud-builders/gcloud'
   args: ['run', 'deploy', 'ingestion-helper-service', '--image', '${_AR_REPO_URL}/datacommons-ingestion-helper:${_VERSION}', '--region', '${_LOCATION}', '--project', '${_PROJECT_ID}', '--no-allow-unauthenticated', '--timeout', '60m', '--set-env-vars', 'PROJECT_ID=${_PROJECT_ID},LOCATION=${_LOCATION},SPANNER_PROJECT_ID=${_SPANNER_PROJECT_ID},SPANNER_INSTANCE_ID=${_SPANNER_INSTANCE_ID},SPANNER_DATABASE_ID=${_SPANNER_DATABASE_ID},SPANNER_GRAPH_DATABASE_ID=${_SPANNER_GRAPH_DATABASE_ID},GCS_BUCKET_ID=${_GCS_BUCKET_ID},BQ_DATASET_ID=${_BQ_DATASET_ID},BQ_SPANNER_CONN_ID=${_BQ_SPANNER_CONN_ID}']
 
-- id: 'import-helper-service'
-  name: 'gcr.io/cloud-builders/gcloud'
-  args: ['run', 'deploy', 'import-helper-service', '--image', '${_AR_REPO_URL}/datacommons-import-helper:${_VERSION}', '--region', '${_LOCATION}', '--project', '${_PROJECT_ID}', '--no-allow-unauthenticated', '--set-env-vars', 'PROJECT_ID=${_PROJECT_ID},LOCATION=${_LOCATION},PROJECT_NUMBER=${_PROJECT_NUMBER},GCS_BUCKET_ID=${_GCS_BUCKET_ID}']
-
 - id: 'import-automation-workflow'
   name: 'gcr.io/cloud-builders/gcloud'
   args: ['workflows', 'deploy', 'import-automation-workflow', '--project', '${_PROJECT_ID}', '--location', '${_LOCATION}', '--source', 'import-automation-workflow.yaml', '--set-env-vars', 'LOCATION=${_LOCATION},GCS_BUCKET_ID=${_GCS_BUCKET_ID},GCS_MOUNT_BUCKET=${_GCS_MOUNT_BUCKET},PROJECT_NUMBER=${_PROJECT_NUMBER}']