Skip to main content

AWS Services Terraform Examples

© 2026 Stephen Adei. All rights reserved. All content on this site is the intellectual property of Stephen Adei. See License for terms of use and attribution.

This section contains Terraform code examples for implementing additional AWS services as outlined in AWS Services Analysis.

1. AWS Lambda - Event-Driven Processing​

A. S3 Event-Driven Ingestion​

# Trigger ETL when new CSV files arrive in Bronze layer

resource "aws_lambda_function" "s3_trigger_etl" {
  function_name = "platform-s3-trigger-etl"
  handler      = "lambda_function.lambda_handler"
  runtime      = "python3.11"
  role         = aws_iam_role.lambda_trigger_role.arn

  environment {
    variables = {
      GLUE_JOB_NAME = aws_glue_job.transaction_etl.name
    }
  }
}

resource "aws_s3_bucket_notification" "bronze_ingestion" {
  bucket = aws_s3_bucket.data_lake_buckets["platform-raw"].id

  lambda_function {
    lambda_function_arn = aws_lambda_function.s3_trigger_etl.arn
    events              = ["s3:ObjectCreated:*"]
    filter_prefix       = "bronze/mortgages/transactions/"
    filter_suffix       = ".csv"
  }
}

B. Post-ETL Validation & Promotion​

# Lambda to validate ETL output and promote to current/

resource "aws_lambda_function" "validate_and_promote" {
  function_name = "platform-validate-promote"
  # Validates _SUCCESS marker, row counts, schema compliance
  # Updates _LATEST.json and copies to current/ prefix
}

C. Data Quality Monitoring​

# Lambda to analyze quarantine patterns and generate reports

resource "aws_lambda_function" "quarantine_analyzer" {
  function_name = "platform-quarantine-analyzer"
  # Runs after each ETL, analyzes quarantine reasons
  # Updates DynamoDB with quality metrics
}

2. AWS Glue - Enhanced Usage​

A. Glue Data Catalog Tables​

resource "aws_glue_catalog_database" "platform_data_lake" {
  name = "platform_data_lake"
}

resource "aws_glue_catalog_table" "silver_transactions" {
  name          = "silver_transactions"
  database_name = aws_glue_catalog_database.platform_data_lake.name

  storage_descriptor {
    location      = "s3://platform-processed/silver/mortgages/transactions/"
    input_format  = "org.apache.hadoop.mapred.TextInputFormat"
    output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
    serde_info {
      serialization_library = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"
    }
    columns {
      name = "transaction_id"
      type = "string"
    }
    columns {
      name = "customer_id"
      type = "string"
    }
    columns {
      name = "transaction_amount"
      type = "decimal(18,2)"
    }
    columns {
      name = "currency"
      type = "string"
    }
    columns {
      name = "transaction_timestamp"
      type = "timestamp"
    }
  }

  partition_keys {
    name = "event_year"
    type = "int"
  }
  partition_keys {
    name = "event_month"
    type = "string"
  }
}

B. Glue Crawlers​

resource "aws_glue_crawler" "silver_crawler" {
  database_name = aws_glue_catalog_database.platform_data_lake.name
  name          = "platform-silver-crawler"
  role          = aws_iam_role.glue_crawler_role.arn

  s3_target {
    path = "s3://platform-processed/silver/mortgages/transactions/"
  }

  schema_change_policy {
    delete_behavior = "LOG"
    update_behavior  = "UPDATE_IN_DATABASE"
  }
}

C. Glue Workflows​

resource "aws_glue_workflow" "transaction_pipeline" {
  name = "platform-transaction-pipeline"
  # Orchestrates: Crawl → ETL → Validate → Promote
}

3. Amazon Athena - SQL Analytics​

A. Athena Workgroup & Configuration​

resource "aws_athena_workgroup" "platform_analytics" {
  name = "platform-analytics"

  configuration {
    enforce_workgroup_configuration    = true
    publish_cloudwatch_metrics_enabled = true

    result_configuration {
      output_location = "s3://platform-artifacts/athena-results/"
      encryption_configuration {
        encryption_option = "SSE_S3"
      }
    }

    engine_version {
      selected_engine_version = "Athena engine version 3"
    }
  }
}

B. Query Result Bucket​

# Already have platform-artifacts bucket, but should add explicit Athena results path
# s3://platform-artifacts/athena-results/

4. Amazon DynamoDB - Metadata & State Management​

A. ETL Run Metadata Table​

resource "aws_dynamodb_table" "etl_runs" {
  name           = "platform-etl-runs"
  billing_mode   = "PAY_PER_REQUEST"
  hash_key       = "run_id"
  range_key      = "dataset_name"

  attribute {
    name = "run_id"
    type = "S"
  }
  attribute {
    name = "dataset_name"
    type = "S"
  }

  ttl {
    attribute_name = "ttl"
    enabled        = true
  }
}

B. Data Quality Metrics Table​

resource "aws_dynamodb_table" "data_quality_metrics" {
  name         = "platform-data-quality"
  billing_mode = "PAY_PER_REQUEST"
  hash_key     = "dataset_name"
  range_key    = "metric_date"

  attribute {
    name = "dataset_name"
    type = "S"
  }
  attribute {
    name = "metric_date"
    type = "S"
  }
}

C. Schema Registry​

resource "aws_dynamodb_table" "schema_registry" {
  name         = "platform-schema-registry"
  billing_mode = "PAY_PER_REQUEST"
  hash_key     = "dataset_name"
  range_key    = "schema_version"

  attribute {
    name = "dataset_name"
    type = "S"
  }
  attribute {
    name = "schema_version"
    type = "S"
  }
}

5. Amazon Aurora - Operational & Reference Data​

A. Reference Data Management​

resource "aws_rds_cluster" "platform_reference_data" {
  cluster_identifier      = "platform-reference-data"
  engine                   = "aurora-postgresql"
  engine_version           = "15.3"
  database_name            = "platform_ref"
  master_username          = "platform_admin"
  master_password          = var.db_password
  backup_retention_period   = 7
  preferred_backup_window  = "03:00-04:00"
  db_subnet_group_name     = aws_db_subnet_group.platform.name
  vpc_security_group_ids   = [aws_security_group.rds.id]
  skip_final_snapshot      = true
}

See also​

© 2026 Stephen Adei•CC BY 4.0