{ "version": 4, "terraform_version": "1.8.1", "serial": 53, "lineage": "a7afa363-ceb3-3da5-99b5-5c54dec39deb", "outputs": {}, "resources": [ { "mode": "managed", "type": "aws_athena_workgroup", "name": "athena_workgroup", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "arn": "arn:aws:athena:us-east-1:613161579745:workgroup/development", "configuration": [ { "bytes_scanned_cutoff_per_query": 0, "enforce_workgroup_configuration": true, "engine_version": [ { "effective_engine_version": "Athena engine version 3", "selected_engine_version": "AUTO" } ], "execution_role": "", "publish_cloudwatch_metrics_enabled": true, "requester_pays_enabled": false, "result_configuration": [ { "acl_configuration": [], "encryption_configuration": [], "expected_bucket_owner": "", "output_location": "s3://athena-results-613161579745-rc-s1201682/output/" } ] } ], "description": "", "force_destroy": true, "id": "development", "name": "development", "state": "ENABLED", "tags": null, "tags_all": {} }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_s3_bucket.athena_results" ] } ] }, { "mode": "managed", "type": "aws_glue_catalog_database", "name": "datalake_db_processed_zone", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "arn": "arn:aws:glue:us-east-1:613161579745:database/datalake_processed_613161579745_rc_s1201682", "catalog_id": "613161579745", "create_table_default_permission": [ { "permissions": [ "ALL" ], "principal": [ { "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" } ] } ], "description": "", "federated_database": [], "id": "613161579745:datalake_processed_613161579745_rc_s1201682", "location_uri": "", "name": "datalake_processed_613161579745_rc_s1201682", "parameters": null, "tags": null, "tags_all": {}, "target_database": [] }, "sensitive_attributes": [], "private": "bnVsbA==" } ] }, { "mode": "managed", "type": "aws_glue_catalog_database", "name": "datalake_db_raw_zone", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "arn": "arn:aws:glue:us-east-1:613161579745:database/datalake_raw_613161579745_rc_s1201682", "catalog_id": "613161579745", "create_table_default_permission": [ { "permissions": [ "ALL" ], "principal": [ { "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" } ] } ], "description": "", "federated_database": [], "id": "613161579745:datalake_raw_613161579745_rc_s1201682", "location_uri": "", "name": "datalake_raw_613161579745_rc_s1201682", "parameters": {}, "tags": {}, "tags_all": {}, "target_database": [] }, "sensitive_attributes": [], "private": "bnVsbA==" } ] }, { "mode": "managed", "type": "aws_glue_crawler", "name": "glue_crawler_raw_zone", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "arn": "arn:aws:glue:us-east-1:613161579745:crawler/gc-raw-613161579745-rc-s1201682", "catalog_target": [], "classifiers": null, "configuration": "", "database_name": "datalake_raw_613161579745_rc_s1201682", "delta_target": [], "description": "", "dynamodb_target": [], "hudi_target": [], "iceberg_target": [], "id": "gc-raw-613161579745-rc-s1201682", "jdbc_target": [], "lake_formation_configuration": [ { "account_id": "", "use_lake_formation_credentials": false } ], "lineage_configuration": [ { "crawler_lineage_settings": "DISABLE" } ], "mongodb_target": [], "name": "gc-raw-613161579745-rc-s1201682", "recrawl_policy": [ { "recrawl_behavior": "CRAWL_EVERYTHING" } ], "role": "LabRole", "s3_target": [ { "connection_name": "", "dlq_event_queue_arn": "", "event_queue_arn": "", "exclusions": null, "path": "s3://datalake-raw-613161579745-rc-s1201682/raw-zone/stockdata/", "sample_size": 0 } ], "schedule": "", "schema_change_policy": [ { "delete_behavior": "DEPRECATE_IN_DATABASE", "update_behavior": "UPDATE_IN_DATABASE" } ], "security_configuration": "", "table_prefix": "crawler_", "tags": { "environment": "DEV", "owner": "Rafal Charzewski", "purpose": "UAM Cloud Data Processing" }, "tags_all": { "environment": "DEV", "owner": "Rafal Charzewski", "purpose": "UAM Cloud Data Processing" } }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_glue_catalog_database.datalake_db_raw_zone", "aws_s3_bucket.raw_bucket" ] } ] }, { "mode": "managed", "type": "aws_kinesis_firehose_delivery_stream", "name": "stock_delivery_stream", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 1, "attributes": { "arn": "arn:aws:firehose:us-east-1:613161579745:deliverystream/firehose-613161579745-rc-s1201682", "destination": "extended_s3", "destination_id": "destinationId-000000000001", "elasticsearch_configuration": [], "extended_s3_configuration": [ { "bucket_arn": "arn:aws:s3:::datalake-raw-613161579745-rc-s1201682", "buffering_interval": 60, "buffering_size": 1, "cloudwatch_logging_options": [ { "enabled": false, "log_group_name": "", "log_stream_name": "" } ], "compression_format": "UNCOMPRESSED", "custom_time_zone": "UTC", "data_format_conversion_configuration": [], "dynamic_partitioning_configuration": [], "error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", "file_extension": "", "kms_key_arn": "", "prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", "processing_configuration": [ { "enabled": false, "processors": [] } ], "role_arn": "arn:aws:iam::613161579745:role/LabRole", "s3_backup_configuration": [], "s3_backup_mode": "Disabled" } ], "http_endpoint_configuration": [], "id": "arn:aws:firehose:us-east-1:613161579745:deliverystream/firehose-613161579745-rc-s1201682", "kinesis_source_configuration": [ { "kinesis_stream_arn": "arn:aws:kinesis:us-east-1:613161579745:stream/cryptostock-613161579745-rc-s1201682", "role_arn": "arn:aws:iam::613161579745:role/LabRole" } ], "msk_source_configuration": [], "name": "firehose-613161579745-rc-s1201682", "opensearch_configuration": [], "opensearchserverless_configuration": [], "redshift_configuration": [], "server_side_encryption": [ { "enabled": false, "key_arn": "", "key_type": "AWS_OWNED_CMK" } ], "snowflake_configuration": [], "splunk_configuration": [], "tags": {}, "tags_all": {}, "timeouts": null, "version_id": "1" }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", "dependencies": [ "aws_kinesis_stream.cryptostock_stream", "aws_s3_bucket.raw_bucket" ] } ] }, { "mode": "managed", "type": "aws_kinesis_stream", "name": "cryptostock_stream", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 1, "attributes": { "arn": "arn:aws:kinesis:us-east-1:613161579745:stream/cryptostock-613161579745-rc-s1201682", "encryption_type": "NONE", "enforce_consumer_deletion": true, "id": "arn:aws:kinesis:us-east-1:613161579745:stream/cryptostock-613161579745-rc-s1201682", "kms_key_id": "", "name": "cryptostock-613161579745-rc-s1201682", "retention_period": 24, "shard_count": 1, "shard_level_metrics": [ "IncomingBytes", "IncomingRecords", "OutgoingBytes", "OutgoingRecords" ], "stream_mode_details": [ { "stream_mode": "PROVISIONED" } ], "tags": { "Environment": "DEV", "Owner": "Rafal Charzewski", "Purpose": "UAM Cloud Data Processing" }, "tags_all": { "Environment": "DEV", "Owner": "Rafal Charzewski", "Purpose": "UAM Cloud Data Processing" }, "timeouts": null }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" } ] }, { "mode": "managed", "type": "aws_lambda_function", "name": "etl_post_processing", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "architectures": [ "x86_64" ], "arn": "arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682", "code_sha256": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=", "code_signing_config_arn": "", "dead_letter_config": [], "description": "", "environment": [], "ephemeral_storage": [ { "size": 512 } ], "file_system_config": [], "filename": "../lambda/lambda_definition.zip", "function_name": "etl-post-processing-613161579745-rc-s1201682", "handler": "lambda_definition.etl_function", "id": "etl-post-processing-613161579745-rc-s1201682", "image_config": [], "image_uri": "", "invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682/invocations", "kms_key_arn": "", "last_modified": "2024-06-19T22:46:02.439+0000", "layers": [ "arn:aws:lambda:us-east-1:613161579745:layer:aws_wrangler_613161579745_rc_s1201682:1" ], "logging_config": [ { "application_log_level": "", "log_format": "Text", "log_group": "/aws/lambda/etl-post-processing-613161579745-rc-s1201682", "system_log_level": "" } ], "memory_size": 512, "package_type": "Zip", "publish": false, "qualified_arn": "arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682:$LATEST", "qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682:$LATEST/invocations", "replace_security_groups_on_destroy": null, "replacement_security_group_ids": null, "reserved_concurrent_executions": -1, "role": "arn:aws:iam::613161579745:role/LabRole", "runtime": "python3.8", "s3_bucket": null, "s3_key": null, "s3_object_version": null, "signing_job_arn": "", "signing_profile_version_arn": "", "skip_destroy": false, "snap_start": [], "source_code_hash": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=", "source_code_size": 884, "tags": null, "tags_all": {}, "timeout": 300, "timeouts": null, "tracing_config": [ { "mode": "PassThrough" } ], "version": "$LATEST", "vpc_config": [] }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", "dependencies": [ "aws_lambda_layer_version.aws_wrangler" ] } ] }, { "mode": "managed", "type": "aws_lambda_layer_version", "name": "aws_wrangler", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "arn": "arn:aws:lambda:us-east-1:613161579745:layer:aws_wrangler_613161579745_rc_s1201682:1", "code_sha256": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=", "compatible_architectures": null, "compatible_runtimes": [ "python3.8" ], "created_date": "2024-06-19T22:46:02.186+0000", "description": "", "filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip", "id": "arn:aws:lambda:us-east-1:613161579745:layer:aws_wrangler_613161579745_rc_s1201682:1", "layer_arn": "arn:aws:lambda:us-east-1:613161579745:layer:aws_wrangler_613161579745_rc_s1201682", "layer_name": "aws_wrangler_613161579745_rc_s1201682", "license_info": "", "s3_bucket": null, "s3_key": null, "s3_object_version": null, "signing_job_arn": "", "signing_profile_version_arn": "", "skip_destroy": false, "source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=", "source_code_size": 43879070, "version": "1" }, "sensitive_attributes": [], "private": "bnVsbA==" } ] }, { "mode": "managed", "type": "aws_lambda_permission", "name": "allow_bucket", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "action": "lambda:InvokeFunction", "event_source_token": null, "function_name": "arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682", "function_url_auth_type": null, "id": "AllowExecutionFromS3Bucket", "principal": "s3.amazonaws.com", "principal_org_id": null, "qualifier": "", "source_account": null, "source_arn": "arn:aws:s3:::datalake-raw-613161579745-rc-s1201682", "statement_id": "AllowExecutionFromS3Bucket", "statement_id_prefix": "" }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_lambda_function.etl_post_processing", "aws_lambda_layer_version.aws_wrangler", "aws_s3_bucket.raw_bucket" ] } ] }, { "mode": "managed", "type": "aws_s3_bucket", "name": "athena_results", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "acceleration_status": "", "acl": null, "arn": "arn:aws:s3:::athena-results-613161579745-rc-s1201682", "bucket": "athena-results-613161579745-rc-s1201682", "bucket_domain_name": "athena-results-613161579745-rc-s1201682.s3.amazonaws.com", "bucket_prefix": "", "bucket_regional_domain_name": "athena-results-613161579745-rc-s1201682.s3.us-east-1.amazonaws.com", "cors_rule": [], "force_destroy": true, "grant": [ { "id": "c8f9aeae204fe9ab20afa7fb31e38920a3327db76890550c9eede7a1ef6c47f7", "permissions": [ "FULL_CONTROL" ], "type": "CanonicalUser", "uri": "" } ], "hosted_zone_id": "Z3AQBSTGFYJSTF", "id": "athena-results-613161579745-rc-s1201682", "lifecycle_rule": [], "logging": [], "object_lock_configuration": [], "object_lock_enabled": false, "policy": "", "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", "server_side_encryption_configuration": [ { "rule": [ { "apply_server_side_encryption_by_default": [ { "kms_master_key_id": "", "sse_algorithm": "AES256" } ], "bucket_key_enabled": false } ] } ], "tags": { "environment": "DEV", "owner": "Rafal Charzewski", "purpose": "UAM Cloud Data Processing" }, "tags_all": { "environment": "DEV", "owner": "Rafal Charzewski", "purpose": "UAM Cloud Data Processing" }, "timeouts": null, "versioning": [ { "enabled": false, "mfa_delete": false } ], "website": [], "website_domain": null, "website_endpoint": null }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" } ] }, { "mode": "managed", "type": "aws_s3_bucket", "name": "processed_bucket", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "acceleration_status": "", "acl": null, "arn": "arn:aws:s3:::datalake-processed-613161579745-rc-s1201682", "bucket": "datalake-processed-613161579745-rc-s1201682", "bucket_domain_name": "datalake-processed-613161579745-rc-s1201682.s3.amazonaws.com", "bucket_prefix": "", "bucket_regional_domain_name": "datalake-processed-613161579745-rc-s1201682.s3.us-east-1.amazonaws.com", "cors_rule": [], "force_destroy": true, "grant": [ { "id": "c8f9aeae204fe9ab20afa7fb31e38920a3327db76890550c9eede7a1ef6c47f7", "permissions": [ "FULL_CONTROL" ], "type": "CanonicalUser", "uri": "" } ], "hosted_zone_id": "Z3AQBSTGFYJSTF", "id": "datalake-processed-613161579745-rc-s1201682", "lifecycle_rule": [], "logging": [], "object_lock_configuration": [], "object_lock_enabled": false, "policy": "", "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", "server_side_encryption_configuration": [ { "rule": [ { "apply_server_side_encryption_by_default": [ { "kms_master_key_id": "", "sse_algorithm": "AES256" } ], "bucket_key_enabled": false } ] } ], "tags": { "Environment": "PROD", "Purpose": "UAM Cloud Data Processing" }, "tags_all": { "Environment": "PROD", "Purpose": "UAM Cloud Data Processing" }, "timeouts": null, "versioning": [ { "enabled": false, "mfa_delete": false } ], "website": [], "website_domain": null, "website_endpoint": null }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" } ] }, { "mode": "managed", "type": "aws_s3_bucket", "name": "raw_bucket", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "acceleration_status": "", "acl": null, "arn": "arn:aws:s3:::datalake-raw-613161579745-rc-s1201682", "bucket": "datalake-raw-613161579745-rc-s1201682", "bucket_domain_name": "datalake-raw-613161579745-rc-s1201682.s3.amazonaws.com", "bucket_prefix": "", "bucket_regional_domain_name": "datalake-raw-613161579745-rc-s1201682.s3.us-east-1.amazonaws.com", "cors_rule": [], "force_destroy": true, "grant": [ { "id": "c8f9aeae204fe9ab20afa7fb31e38920a3327db76890550c9eede7a1ef6c47f7", "permissions": [ "FULL_CONTROL" ], "type": "CanonicalUser", "uri": "" } ], "hosted_zone_id": "Z3AQBSTGFYJSTF", "id": "datalake-raw-613161579745-rc-s1201682", "lifecycle_rule": [], "logging": [], "object_lock_configuration": [], "object_lock_enabled": false, "policy": "", "region": "us-east-1", "replication_configuration": [], "request_payer": "BucketOwner", "server_side_encryption_configuration": [ { "rule": [ { "apply_server_side_encryption_by_default": [ { "kms_master_key_id": "", "sse_algorithm": "AES256" } ], "bucket_key_enabled": false } ] } ], "tags": { "Environment": "DEV", "Purpose": "UAM Cloud Data Processing" }, "tags_all": { "Environment": "DEV", "Purpose": "UAM Cloud Data Processing" }, "timeouts": null, "versioning": [ { "enabled": false, "mfa_delete": false } ], "website": [], "website_domain": null, "website_endpoint": null }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" } ] }, { "mode": "managed", "type": "aws_s3_bucket_lifecycle_configuration", "name": "athena_results_lifecycle", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "bucket": "athena-results-613161579745-rc-s1201682", "expected_bucket_owner": "", "id": "athena-results-613161579745-rc-s1201682", "rule": [ { "abort_incomplete_multipart_upload": [], "expiration": [ { "date": "", "days": 1, "expired_object_delete_marker": false } ], "filter": [ { "and": [], "object_size_greater_than": "", "object_size_less_than": "", "prefix": "", "tag": [] } ], "id": "standard-expiration", "noncurrent_version_expiration": [], "noncurrent_version_transition": [], "prefix": "", "status": "Enabled", "transition": [] } ], "timeouts": null }, "sensitive_attributes": [], "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=", "dependencies": [ "aws_s3_bucket.athena_results" ] } ] }, { "mode": "managed", "type": "aws_s3_bucket_notification", "name": "trigger_etl_lambda", "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", "instances": [ { "schema_version": 0, "attributes": { "bucket": "datalake-raw-613161579745-rc-s1201682", "eventbridge": false, "id": "datalake-raw-613161579745-rc-s1201682", "lambda_function": [ { "events": [ "s3:ObjectCreated:*" ], "filter_prefix": "raw-zone/", "filter_suffix": "", "id": "tf-s3-lambda-20240619224612776100000001", "lambda_function_arn": "arn:aws:lambda:us-east-1:613161579745:function:etl-post-processing-613161579745-rc-s1201682" } ], "queue": [], "topic": [] }, "sensitive_attributes": [], "private": "bnVsbA==", "dependencies": [ "aws_lambda_function.etl_post_processing", "aws_lambda_layer_version.aws_wrangler", "aws_lambda_permission.allow_bucket", "aws_s3_bucket.raw_bucket" ] } ] } ], "check_results": null }