diff --git a/labs/sql_queries.sql b/labs/sql_queries.sql new file mode 100644 index 0000000..4f57721 --- /dev/null +++ b/labs/sql_queries.sql @@ -0,0 +1,12 @@ +WITH cte AS( + SELECT DISTINCT date_format(from_unixtime(transaction_ts),'%Y-%m-%dT%H') AS HourlyBucket, + RANK() OVER(PARTITION BY (date_format(from_unixtime(transaction_ts),'%Y-%m-%dT%H')), symbol, type ORDER BY dollar_amount DESC) AS rnk, + transaction_ts, symbol, price, amount, dollar_amount, type, trans_id, year, month, day, hour + FROM "datalake_raw_534534002841_ab_1201680"."crawler_stockdata" + ORDER BY HourlyBucket +) + +SELECT HourlyBucket, rnk, transaction_ts, symbol, price, amount, dollar_amount, type, trans_id, year, month, day, hour +FROM cte +WHERE rnk = 1 +ORDER BY HourlyBucket, symbol, type \ No newline at end of file diff --git a/labs/terraform/athena.tf b/labs/terraform/athena.tf new file mode 100644 index 0000000..a7c275b --- /dev/null +++ b/labs/terraform/athena.tf @@ -0,0 +1,28 @@ +resource "aws_s3_bucket" "athena_results" { + bucket = "athena-results-${var.account_number}-${var.student_initials}-${var.student_index_no}" + force_destroy = true + tags = merge(local.common_tags, ) +} + +resource "aws_s3_bucket_lifecycle_configuration" "athena_results_lifecycle" { + bucket = aws_s3_bucket.athena_results.id + + rule { + id = "standard-expiration" + status = "Enabled" + expiration { + days=1 + } + } +} + +resource "aws_athena_workgroup" "athena_workgroup" { + name = "development" + configuration { + enforce_workgroup_configuration = true + result_configuration { + output_location = "s3://${aws_s3_bucket.athena_results.bucket}/output/" + } + } + force_destroy = true +} diff --git a/labs/terraform/glue.tf b/labs/terraform/glue.tf new file mode 100644 index 0000000..92665ba --- /dev/null +++ b/labs/terraform/glue.tf @@ -0,0 +1,16 @@ +resource "aws_glue_catalog_database" "datalake_db_processed_zone" { + name = "datalake_processed_${var.account_number}_${var.student_initials}_${var.student_index_no}" +} + +resource "aws_glue_crawler" "glue_crawler_raw_zone" { + database_name = aws_glue_catalog_database.datalake_db_raw_zone.name + name = "gc-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}" + role = var.lab_role_arn + table_prefix = "crawler_" + + s3_target { + path = "s3://${aws_s3_bucket.raw_bucket.bucket}/raw-zone/stockdata/" + } + + tags = merge(local.common_tags, ) +} diff --git a/labs/terraform/lambda.tf b/labs/terraform/lambda.tf new file mode 100644 index 0000000..0da76f5 --- /dev/null +++ b/labs/terraform/lambda.tf @@ -0,0 +1,36 @@ +resource "aws_lambda_layer_version" "aws_wrangler" { + filename = "../lambda/awswrangler-layer-2.7.0-py3.8.zip" + layer_name = "aws_wrangler_${var.account_number}_${var.student_initials}_${var.student_index_no}" + source_code_hash = "${filebase64sha256("../lambda/awswrangler-layer-2.7.0-py3.8.zip")}" + compatible_runtimes = ["python3.8"] +} + +resource "aws_lambda_function" "etl_post_processing" { + function_name = "etl-post-processing-${var.account_number}-${var.student_initials}-${var.student_index_no}" + filename = "../lambda/lambda_definition.zip" + handler = "lambda_definition.etl_function" + runtime = "python3.8" + role = var.lab_role_arn + timeout = 300 + memory_size = 512 + source_code_hash= filebase64sha256("../lambda/lambda_definition.zip") + layers = ["${aws_lambda_layer_version.aws_wrangler.arn}"] +} + +resource "aws_lambda_permission" "allow_bucket" { + statement_id = "AllowExecutionFromS3Bucket" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.etl_post_processing.arn + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.raw_bucket.arn +} + +resource "aws_s3_bucket_notification" "trigger_etl_lambda" { + bucket = aws_s3_bucket.raw_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.etl_post_processing.arn + events = ["s3:ObjectCreated:*"] + filter_prefix = "raw-zone/" + } + depends_on = [aws_lambda_permission.allow_bucket] +} \ No newline at end of file diff --git a/labs/terraform/terraform.tfstate b/labs/terraform/terraform.tfstate index 6a67cdd..3f884f0 100644 --- a/labs/terraform/terraform.tfstate +++ b/labs/terraform/terraform.tfstate @@ -1,327 +1,9 @@ { "version": 4, "terraform_version": "1.8.1", - "serial": 55, + "serial": 145, "lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2", "outputs": {}, - "resources": [ - { - "mode": "managed", - "type": "aws_glue_catalog_database", - "name": "datalake_db_raw_zone", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680", - "catalog_id": "534534002841", - "create_table_default_permission": [ - { - "permissions": [ - "ALL" - ], - "principal": [ - { - "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" - } - ] - } - ], - "description": "", - "federated_database": [], - "id": "534534002841:datalake_raw_534534002841_ab_1201680", - "location_uri": "", - "name": "datalake_raw_534534002841_ab_1201680", - "parameters": null, - "tags": null, - "tags_all": {}, - "target_database": [] - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - }, - { - "mode": "managed", - "type": "aws_kinesis_firehose_delivery_stream", - "name": "stock_delivery_stream", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", - "destination": "extended_s3", - "destination_id": "destinationId-000000000001", - "elasticsearch_configuration": [], - "extended_s3_configuration": [ - { - "bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", - "buffering_interval": 60, - "buffering_size": 1, - "cloudwatch_logging_options": [ - { - "enabled": false, - "log_group_name": "", - "log_stream_name": "" - } - ], - "compression_format": "UNCOMPRESSED", - "custom_time_zone": "UTC", - "data_format_conversion_configuration": [], - "dynamic_partitioning_configuration": [], - "error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", - "file_extension": "", - "kms_key_arn": "", - "prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", - "processing_configuration": [ - { - "enabled": false, - "processors": [] - } - ], - "role_arn": "arn:aws:iam::534534002841:role/LabRole", - "s3_backup_configuration": [], - "s3_backup_mode": "Disabled" - } - ], - "http_endpoint_configuration": [], - "id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", - "kinesis_source_configuration": [ - { - "kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", - "role_arn": "arn:aws:iam::534534002841:role/LabRole" - } - ], - "msk_source_configuration": [], - "name": "firehose-534534002841-ab-1201680", - "opensearch_configuration": [], - "opensearchserverless_configuration": [], - "redshift_configuration": [], - "server_side_encryption": [ - { - "enabled": false, - "key_arn": "", - "key_type": "AWS_OWNED_CMK" - } - ], - "snowflake_configuration": [], - "splunk_configuration": [], - "tags": null, - "tags_all": {}, - "timeouts": null, - "version_id": "1" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", - "dependencies": [ - "aws_kinesis_stream.cryptostock_stream", - "aws_s3_bucket.raw_bucket" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_kinesis_stream", - "name": "cryptostock_stream", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", - "encryption_type": "NONE", - "enforce_consumer_deletion": true, - "id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", - "kms_key_id": "", - "name": "cryptostock-534534002841-ab-1201680", - "retention_period": 24, - "shard_count": 1, - "shard_level_metrics": [ - "IncomingBytes", - "IncomingRecords", - "OutgoingBytes", - "OutgoingRecords" - ], - "stream_mode_details": [ - { - "stream_mode": "PROVISIONED" - } - ], - "tags": { - "Environment": "DEV", - "Owner": "Aron Boguszewski", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Owner": "Aron Boguszewski", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket", - "name": "processed_bucket", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "acceleration_status": "", - "acl": null, - "arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680", - "bucket": "datalake-processed-534534002841-ab-1201680", - "bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com", - "bucket_prefix": "", - "bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", - "cors_rule": [], - "force_destroy": true, - "grant": [ - { - "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", - "permissions": [ - "FULL_CONTROL" - ], - "type": "CanonicalUser", - "uri": "" - } - ], - "hosted_zone_id": "Z3AQBSTGFYJSTF", - "id": "datalake-processed-534534002841-ab-1201680", - "lifecycle_rule": [], - "logging": [], - "object_lock_configuration": [], - "object_lock_enabled": false, - "policy": "", - "region": "us-east-1", - "replication_configuration": [], - "request_payer": "BucketOwner", - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": "", - "sse_algorithm": "AES256" - } - ], - "bucket_key_enabled": false - } - ] - } - ], - "tags": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null, - "versioning": [ - { - "enabled": false, - "mfa_delete": false - } - ], - "website": [], - "website_domain": null, - "website_endpoint": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket", - "name": "raw_bucket", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "acceleration_status": "", - "acl": null, - "arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", - "bucket": "datalake-raw-534534002841-ab-1201680", - "bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com", - "bucket_prefix": "", - "bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", - "cors_rule": [], - "force_destroy": true, - "grant": [ - { - "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", - "permissions": [ - "FULL_CONTROL" - ], - "type": "CanonicalUser", - "uri": "" - } - ], - "hosted_zone_id": "Z3AQBSTGFYJSTF", - "id": "datalake-raw-534534002841-ab-1201680", - "lifecycle_rule": [], - "logging": [], - "object_lock_configuration": [], - "object_lock_enabled": false, - "policy": "", - "region": "us-east-1", - "replication_configuration": [], - "request_payer": "BucketOwner", - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": "", - "sse_algorithm": "AES256" - } - ], - "bucket_key_enabled": false - } - ] - } - ], - "tags": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null, - "versioning": [ - { - "enabled": false, - "mfa_delete": false - } - ], - "website": [], - "website_domain": null, - "website_endpoint": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] - } - ], + "resources": [], "check_results": null } diff --git a/labs/terraform/terraform.tfstate.backup b/labs/terraform/terraform.tfstate.backup index f3818e2..daf04b0 100644 --- a/labs/terraform/terraform.tfstate.backup +++ b/labs/terraform/terraform.tfstate.backup @@ -1,9 +1,798 @@ { "version": 4, "terraform_version": "1.8.1", - "serial": 49, + "serial": 130, "lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2", "outputs": {}, - "resources": [], + "resources": [ + { + "mode": "managed", + "type": "aws_athena_workgroup", + "name": "athena_workgroup", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:athena:us-east-1:534534002841:workgroup/development", + "configuration": [ + { + "bytes_scanned_cutoff_per_query": 0, + "enforce_workgroup_configuration": true, + "engine_version": [ + { + "effective_engine_version": "Athena engine version 3", + "selected_engine_version": "AUTO" + } + ], + "execution_role": "", + "publish_cloudwatch_metrics_enabled": true, + "requester_pays_enabled": false, + "result_configuration": [ + { + "acl_configuration": [], + "encryption_configuration": [], + "expected_bucket_owner": "", + "output_location": "s3://athena-results-534534002841-ab-1201680/output/" + } + ] + } + ], + "description": "", + "force_destroy": true, + "id": "development", + "name": "development", + "state": "ENABLED", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.athena_results" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_glue_catalog_database", + "name": "datalake_db_processed_zone", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_processed_534534002841_ab_1201680", + "catalog_id": "534534002841", + "create_table_default_permission": [ + { + "permissions": [ + "ALL" + ], + "principal": [ + { + "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" + } + ] + } + ], + "description": "", + "federated_database": [], + "id": "534534002841:datalake_processed_534534002841_ab_1201680", + "location_uri": "", + "name": "datalake_processed_534534002841_ab_1201680", + "parameters": null, + "tags": null, + "tags_all": {}, + "target_database": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_glue_catalog_database", + "name": "datalake_db_raw_zone", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680", + "catalog_id": "534534002841", + "create_table_default_permission": [ + { + "permissions": [ + "ALL" + ], + "principal": [ + { + "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" + } + ] + } + ], + "description": "", + "federated_database": [], + "id": "534534002841:datalake_raw_534534002841_ab_1201680", + "location_uri": "", + "name": "datalake_raw_534534002841_ab_1201680", + "parameters": null, + "tags": null, + "tags_all": {}, + "target_database": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_glue_crawler", + "name": "glue_crawler_raw_zone", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:glue:us-east-1:534534002841:crawler/gc-raw-534534002841-ab-1201680", + "catalog_target": [], + "classifiers": null, + "configuration": "", + "database_name": "datalake_raw_534534002841_ab_1201680", + "delta_target": [], + "description": "", + "dynamodb_target": [], + "hudi_target": [], + "iceberg_target": [], + "id": "gc-raw-534534002841-ab-1201680", + "jdbc_target": [], + "lake_formation_configuration": [ + { + "account_id": "", + "use_lake_formation_credentials": false + } + ], + "lineage_configuration": [ + { + "crawler_lineage_settings": "DISABLE" + } + ], + "mongodb_target": [], + "name": "gc-raw-534534002841-ab-1201680", + "recrawl_policy": [ + { + "recrawl_behavior": "CRAWL_EVERYTHING" + } + ], + "role": "LabRole", + "s3_target": [ + { + "connection_name": "", + "dlq_event_queue_arn": "", + "event_queue_arn": "", + "exclusions": null, + "path": "s3://datalake-raw-534534002841-ab-1201680/raw-zone/stockdata/", + "sample_size": 0 + } + ], + "schedule": "", + "schema_change_policy": [ + { + "delete_behavior": "DEPRECATE_IN_DATABASE", + "update_behavior": "UPDATE_IN_DATABASE" + } + ], + "security_configuration": "", + "table_prefix": "crawler_", + "tags": { + "environment": "DEV", + "owner": "Aron Boguszewski", + "purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "environment": "DEV", + "owner": "Aron Boguszewski", + "purpose": "UAM Cloud Data Processing" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_glue_catalog_database.datalake_db_raw_zone", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_kinesis_firehose_delivery_stream", + "name": "stock_delivery_stream", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", + "destination": "extended_s3", + "destination_id": "destinationId-000000000001", + "elasticsearch_configuration": [], + "extended_s3_configuration": [ + { + "bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", + "buffering_interval": 60, + "buffering_size": 1, + "cloudwatch_logging_options": [ + { + "enabled": false, + "log_group_name": "", + "log_stream_name": "" + } + ], + "compression_format": "UNCOMPRESSED", + "custom_time_zone": "UTC", + "data_format_conversion_configuration": [], + "dynamic_partitioning_configuration": [], + "error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", + "file_extension": "", + "kms_key_arn": "", + "prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", + "processing_configuration": [ + { + "enabled": false, + "processors": [] + } + ], + "role_arn": "arn:aws:iam::534534002841:role/LabRole", + "s3_backup_configuration": [], + "s3_backup_mode": "Disabled" + } + ], + "http_endpoint_configuration": [], + "id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", + "kinesis_source_configuration": [ + { + "kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "role_arn": "arn:aws:iam::534534002841:role/LabRole" + } + ], + "msk_source_configuration": [], + "name": "firehose-534534002841-ab-1201680", + "opensearch_configuration": [], + "opensearchserverless_configuration": [], + "redshift_configuration": [], + "server_side_encryption": [ + { + "enabled": false, + "key_arn": "", + "key_type": "AWS_OWNED_CMK" + } + ], + "snowflake_configuration": [], + "splunk_configuration": [], + "tags": null, + "tags_all": {}, + "timeouts": null, + "version_id": "1" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "aws_kinesis_stream.cryptostock_stream", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_kinesis_stream", + "name": "cryptostock_stream", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "encryption_type": "NONE", + "enforce_consumer_deletion": true, + "id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "kms_key_id": "", + "name": "cryptostock-534534002841-ab-1201680", + "retention_period": 24, + "shard_count": 1, + "shard_level_metrics": [ + "IncomingBytes", + "IncomingRecords", + "OutgoingBytes", + "OutgoingRecords" + ], + "stream_mode_details": [ + { + "stream_mode": "PROVISIONED" + } + ], + "tags": { + "Environment": "DEV", + "Owner": "Aron Boguszewski", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Owner": "Aron Boguszewski", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_function", + "name": "etl_post_processing", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architectures": [ + "x86_64" + ], + "arn": "arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680", + "code_signing_config_arn": "", + "dead_letter_config": [], + "description": "", + "environment": [], + "ephemeral_storage": [ + { + "size": 512 + } + ], + "file_system_config": [], + "filename": "../lambda/lambda_definition.zip", + "function_name": "etl-post-processing-534534002841-ab-1201680", + "handler": "lambda_definition.etl_function", + "id": "etl-post-processing-534534002841-ab-1201680", + "image_config": [], + "image_uri": "", + "invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680/invocations", + "kms_key_arn": "", + "last_modified": "2024-05-11T10:53:43.594+0000", + "layers": [ + "arn:aws:lambda:us-east-1:534534002841:layer:aws_wrangler_534534002841_ab_1201680:2" + ], + "logging_config": [ + { + "application_log_level": "", + "log_format": "Text", + "log_group": "/aws/lambda/etl-post-processing-534534002841-ab-1201680", + "system_log_level": "" + } + ], + "memory_size": 512, + "package_type": "Zip", + "publish": false, + "qualified_arn": "arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680:$LATEST", + "qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680:$LATEST/invocations", + "replace_security_groups_on_destroy": null, + "replacement_security_group_ids": null, + "reserved_concurrent_executions": -1, + "role": "arn:aws:iam::534534002841:role/LabRole", + "runtime": "python3.8", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "snap_start": [], + "source_code_hash": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=", + "source_code_size": 884, + "tags": null, + "tags_all": {}, + "timeout": 300, + "timeouts": null, + "tracing_config": [ + { + "mode": "PassThrough" + } + ], + "version": "$LATEST", + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "aws_lambda_layer_version.aws_wrangler" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_layer_version", + "name": "aws_wrangler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:lambda:us-east-1:534534002841:layer:aws_wrangler_534534002841_ab_1201680:2", + "compatible_architectures": null, + "compatible_runtimes": [ + "python3.8" + ], + "created_date": "2024-05-11T10:53:43.233+0000", + "description": "", + "filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip", + "id": "arn:aws:lambda:us-east-1:534534002841:layer:aws_wrangler_534534002841_ab_1201680:2", + "layer_arn": "arn:aws:lambda:us-east-1:534534002841:layer:aws_wrangler_534534002841_ab_1201680", + "layer_name": "aws_wrangler_534534002841_ab_1201680", + "license_info": "", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=", + "source_code_size": 43879070, + "version": "2" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_permission", + "name": "allow_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "action": "lambda:InvokeFunction", + "event_source_token": null, + "function_name": "arn:aws:lambda:us-east-1:534534002841:function:etl-post-processing-534534002841-ab-1201680", + "function_url_auth_type": null, + "id": "AllowExecutionFromS3Bucket", + "principal": "s3.amazonaws.com", + "principal_org_id": null, + "qualifier": "", + "source_account": null, + "source_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", + "statement_id": "AllowExecutionFromS3Bucket", + "statement_id_prefix": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_lambda_function.etl_post_processing", + "aws_lambda_layer_version.aws_wrangler", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "athena_results", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::athena-results-534534002841-ab-1201680", + "bucket": "athena-results-534534002841-ab-1201680", + "bucket_domain_name": "athena-results-534534002841-ab-1201680.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "athena-results-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "athena-results-534534002841-ab-1201680", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "environment": "DEV", + "owner": "Aron Boguszewski", + "purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "environment": "DEV", + "owner": "Aron Boguszewski", + "purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "processed_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680", + "bucket": "datalake-processed-534534002841-ab-1201680", + "bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "datalake-processed-534534002841-ab-1201680", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "raw_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", + "bucket": "datalake-raw-534534002841-ab-1201680", + "bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "datalake-raw-534534002841-ab-1201680", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_lifecycle_configuration", + "name": "athena_results_lifecycle", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "athena-results-534534002841-ab-1201680", + "expected_bucket_owner": "", + "id": "athena-results-534534002841-ab-1201680", + "rule": [ + { + "abort_incomplete_multipart_upload": [], + "expiration": [ + { + "date": "", + "days": 1, + "expired_object_delete_marker": false + } + ], + "filter": [ + { + "and": [], + "object_size_greater_than": "", + "object_size_less_than": "", + "prefix": "", + "tag": [] + } + ], + "id": "standard-expiration", + "noncurrent_version_expiration": [], + "noncurrent_version_transition": [], + "prefix": "", + "status": "Enabled", + "transition": [] + } + ], + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=", + "dependencies": [ + "aws_s3_bucket.athena_results" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_notification", + "name": "trigger_etl_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "datalake-raw-534534002841-ab-1201680", + "eventbridge": false, + "id": "datalake-raw-534534002841-ab-1201680", + "lambda_function": [], + "queue": [], + "topic": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_lambda_function.etl_post_processing", + "aws_lambda_layer_version.aws_wrangler", + "aws_lambda_permission.allow_bucket", + "aws_s3_bucket.raw_bucket" + ] + } + ] + } + ], "check_results": null }