From bc1f2ad333511742add648f3fcf784c4f5a139ae Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Thu, 30 May 2024 09:28:22 +0000 Subject: [PATCH] Dodano nowe pliki i zmiany w kodzie Lambda --- labs/lambda/lambda_definition.py | 18 +- labs/lambda/lambda_definition.zip | Bin 884 -> 985 bytes labs/terraform/terraform.tfstate | 873 ++++++++++++++++++++++- labs/terraform/terraform.tfstate.backup | 891 +----------------------- 4 files changed, 880 insertions(+), 902 deletions(-) diff --git a/labs/lambda/lambda_definition.py b/labs/lambda/lambda_definition.py index 100e432..ba7e6ca 100644 --- a/labs/lambda/lambda_definition.py +++ b/labs/lambda/lambda_definition.py @@ -9,16 +9,19 @@ def etl_function(event, context): processed_zone_prefix = "processed-zone" record = event["Records"][0] - bucket = client_ssm.get_parameter(Name = 's3_processed_bucket_name')['Parameter']['Value'] + raw_bucket = record["s3"]["bucket"]["name"] key = urllib.parse.unquote(record["s3"]["object"]["key"]) event_prefix = key.split('/')[1] - full_src_path = 's3://{bucket}/{key}'.format(bucket=bucket, key=key) + full_src_path = f's3://{raw_bucket}/{key}' + + # Pobierz nazwę bucketu przetworzonych danych z SSM + processed_bucket = client_ssm.get_parameter(Name='s3_processed_bucket_name')['Parameter']['Value'] print(f'Processing key = {full_src_path}') df = wr.s3.read_json(path=full_src_path, lines=True) filename = key.split('/')[-1][-36:] - dest_prefix = f"s3://{bucket}/{processed_zone_prefix}/{event_prefix}" + dest_prefix = f"s3://{processed_bucket}/{processed_zone_prefix}/{event_prefix}" df['transaction_date'] = pd.to_datetime(df['transaction_ts'], unit='s') df['year'] = df['transaction_date'].dt.year @@ -46,11 +49,4 @@ def etl_function(event, context): 'key': key, 'statusCode': 200, 'new_keys': new_keys - } - - -if __name__ == "__main__": - event = "" - context = "" - - response = etl_function(event, context) \ No newline at end of file + } \ No newline at end of file diff --git a/labs/lambda/lambda_definition.zip b/labs/lambda/lambda_definition.zip index 700bc111146736a4b8bd96ed941ecaeec5582849..628af3dfbc0a3fa862996865c2953c88f06ea99e 100644 GIT binary patch delta 925 zcmV;O17iI22H6J~P)h>@6aWAK2mmQozF0KbH4!TV002J+kr*6*!ET%|5WTOo|6tLB zS(QN2m3oN8spnQvQ}u!n@?w)XVQt4Y*|3NY>HqY@IySH@+elfhu$g&pJo7vQ-z%vt zYII%mU95;UY`q$d(zkQtdyDpz=z9%pZ?thd3K$T2y!TPP5I0$N=JAWm@OcJ!R&T|>{&1po>`01 zg+Os5UK(jxxTMS76DvI(C_%m%=suA-ad5>()f}`)@Fu%|84szNx-N_^3q|ZBz$$FE zw@K2?+4e~X+#XMrenNM0$C*ANL5_#Nf6xGHK(ZsV2dP1qwtPgC_~nT1@4sJadeULO zvG{>OwnT#fmg(>ZtSRJDT#^bW5fp&$CM?b~{F~Gb!}*k>(i|?m!gm9ii-%FoZc%TK zN3CcDzJrc`%{JDI(BjEJV>m>*z(=U&f|>M}ZrElztKv0tIsLEk)lHsVZNJ`*Qd4Hm z?xP9@p)c#}lkVN#1N-@n_NUIHRfa80iy^+B3rZ{~0VgOLTj^gd-?Q)%XARCH)Cg`t z=UIphw~S~X;Qh!kwXuWDgguPT;|C0nnJJXC9}ob4F~c8a^HE9;GD%X(+7wW`hPKoK zXjyPc3L-bky!0EgmyNJ)M`c|TUCajCDi87eg45vCV}czDxNzgbPs1cN%1pc8$vQ$# zqX_xBBN&D@LN3MJ?JYrEKrQiumW|rAVTK15j?f@~gho{(
>sKNAo@?B1}<*p(# z^pg*Nk#@t{CoWLhxo4mFU}@)8K8uKr zCAKl&BxT_Ha(#X7n@R`Iv>m+tdi4)bO928O02BZK00;mnR=!v?*)0{{R&2a^&5 zG%olnZ9;Cc+5z}0Z9;Cc+5t@A5M#YF*8xyV1qJ{B000310RUzI003PB00000PJO7~ delta 823 zcmV-71IYZ@2lNIQP)h>@6aWAK2msBll2Y`UP}#-;004Iekr*6*&5oNe5WYvsJ6K%I zsw`|9sfS3M`T|wk8$!r|Vb{B0v$ol6ig@>qZ3vLI+WcV0^UXK&jc3B!PAWrs)inGR zcT8#6@1{d`)@Q}UV*`pXP0s3S>6obS1V3H1-`RHsH38EUbuUW8r3m2-go#Kgg@Jc- zciis?zB(mK&>E_L;}AD5a_p;OMg8hx@IAJbTDJ;?$Qf zU=ZP-25hnZR(Ah6 z&+ZRjkNFe?^r{MVaHH>^cZ}!j>Wz5AbH*7p*kzizz9}kZfab{CRk4xor{OJxw>YC| z9+6&flTtl@%eSBlFl7n;!yH#8wvff4#TI?~z{6n{mx^6Ja6rTYf0F%2DOt!|l2SIh zKjVl$& z^rbz?CL*>*5plXBf@&gSQ_St&Fv10Di652hyVE6q%&7OG5pm_WtMjxQ^u=YWf%_kr zUO0b-TSeM+k*7!pzmAC|N(cKK6GxT~ZW$9~Ob2|736dhlKF7cl&jV<+6;n535691v zi!fVtZnTOYExrGVQ2WR5+JnJt=VGl#@h5w;W5{d;=V9DLybj~K%rWaa5LLKAuKA~_ z#cQ8q!iS5=_fcvqpKY5XkG80J)R!1HfKs-oUu%?&b0aXT3tbS`gr?RT786B6dbEP0 z)uA@b^!l5u0Bv6$9@e&5I$Gwfw~qVWA5cpH0zUv0000080L`wFQuLWn*~S6@0CxtH zcLOvo%;1!vj5OB)D>8h)j5OB)OyLk?y))MVP)h{{000000RRC2W&i*H_yPa`004W> Bh>idN diff --git a/labs/terraform/terraform.tfstate b/labs/terraform/terraform.tfstate index 303ecc1..35b09bf 100644 --- a/labs/terraform/terraform.tfstate +++ b/labs/terraform/terraform.tfstate @@ -1,9 +1,878 @@ { "version": 4, "terraform_version": "1.8.1", - "serial": 289, + "serial": 340, "lineage": "3ca48c2b-bb88-35e7-8a35-5ef0e08daaff", "outputs": {}, - "resources": [], + "resources": [ + { + "mode": "managed", + "type": "aws_athena_workgroup", + "name": "athena_workgroup", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:athena:us-east-1:205216182560:workgroup/development", + "configuration": [ + { + "bytes_scanned_cutoff_per_query": 0, + "enforce_workgroup_configuration": true, + "engine_version": [ + { + "effective_engine_version": "Athena engine version 3", + "selected_engine_version": "AUTO" + } + ], + "execution_role": "", + "publish_cloudwatch_metrics_enabled": true, + "requester_pays_enabled": false, + "result_configuration": [ + { + "acl_configuration": [], + "encryption_configuration": [], + "expected_bucket_owner": "", + "output_location": "s3://athena-results-205216182560-agb-s1201687/output/" + } + ] + } + ], + "description": "", + "force_destroy": true, + "id": "development", + "name": "development", + "state": "ENABLED", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.athena_results" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_glue_catalog_database", + "name": "datalake_db_processed_zone", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_processed_205216182560_agb_s1201687", + "catalog_id": "205216182560", + "create_table_default_permission": [ + { + "permissions": [ + "ALL" + ], + "principal": [ + { + "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" + } + ] + } + ], + "description": "", + "federated_database": [], + "id": "205216182560:datalake_processed_205216182560_agb_s1201687", + "location_uri": "", + "name": "datalake_processed_205216182560_agb_s1201687", + "parameters": null, + "tags": null, + "tags_all": {}, + "target_database": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_glue_catalog_database", + "name": "datalake_db_raw_zone", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_raw_205216182560_agb_s1201687", + "catalog_id": "205216182560", + "create_table_default_permission": [ + { + "permissions": [ + "ALL" + ], + "principal": [ + { + "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" + } + ] + } + ], + "description": "", + "federated_database": [], + "id": "205216182560:datalake_raw_205216182560_agb_s1201687", + "location_uri": "", + "name": "datalake_raw_205216182560_agb_s1201687", + "parameters": null, + "tags": null, + "tags_all": {}, + "target_database": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_glue_crawler", + "name": "glue_crawler_raw_zone", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:glue:us-east-1:205216182560:crawler/gc-raw-205216182560-agb-s1201687", + "catalog_target": [], + "classifiers": null, + "configuration": "", + "database_name": "datalake_raw_205216182560_agb_s1201687", + "delta_target": [], + "description": "", + "dynamodb_target": [], + "hudi_target": [], + "iceberg_target": [], + "id": "gc-raw-205216182560-agb-s1201687", + "jdbc_target": [], + "lake_formation_configuration": [ + { + "account_id": "", + "use_lake_formation_credentials": false + } + ], + "lineage_configuration": [ + { + "crawler_lineage_settings": "DISABLE" + } + ], + "mongodb_target": [], + "name": "gc-raw-205216182560-agb-s1201687", + "recrawl_policy": [ + { + "recrawl_behavior": "CRAWL_EVERYTHING" + } + ], + "role": "LabRole", + "s3_target": [ + { + "connection_name": "", + "dlq_event_queue_arn": "", + "event_queue_arn": "", + "exclusions": null, + "path": "s3://datalake-raw-205216182560-agb-s1201687/raw-zone/stockdata/", + "sample_size": 0 + } + ], + "schedule": "", + "schema_change_policy": [ + { + "delete_behavior": "DEPRECATE_IN_DATABASE", + "update_behavior": "UPDATE_IN_DATABASE" + } + ], + "security_configuration": "", + "table_prefix": "crawler_", + "tags": { + "environment": "DEV", + "owner": "Agnieszka Gąbka-Buszek", + "purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "environment": "DEV", + "owner": "Agnieszka Gąbka-Buszek", + "purpose": "UAM Cloud Data Processing" + } + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_glue_catalog_database.datalake_db_raw_zone", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_iam_policy", + "name": "lambda_policy", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:iam::205216182560:policy/lambda_policy", + "attachment_count": 0, + "description": "", + "id": "arn:aws:iam::205216182560:policy/lambda_policy", + "name": "lambda_policy", + "name_prefix": "", + "path": "/", + "policy": "{\"Statement\":[{\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::datalake-raw-205216182560-agb-s1201687/*\",\"arn:aws:s3:::datalake-processed-205216182560-agb-s1201687/*\"]},{\"Action\":\"ssm:GetParameter\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name\"}],\"Version\":\"2012-10-17\"}", + "policy_id": "ANPAS7R6WOEQMJ2ODN7S5", + "tags": null, + "tags_all": {} + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.processed_bucket", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_kinesis_firehose_delivery_stream", + "name": "stock_delivery_stream", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687", + "destination": "extended_s3", + "destination_id": "destinationId-000000000001", + "elasticsearch_configuration": [], + "extended_s3_configuration": [ + { + "bucket_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687", + "buffering_interval": 60, + "buffering_size": 1, + "cloudwatch_logging_options": [ + { + "enabled": false, + "log_group_name": "", + "log_stream_name": "" + } + ], + "compression_format": "UNCOMPRESSED", + "custom_time_zone": "UTC", + "data_format_conversion_configuration": [], + "dynamic_partitioning_configuration": [], + "error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", + "file_extension": "", + "kms_key_arn": "", + "prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", + "processing_configuration": [ + { + "enabled": false, + "processors": [] + } + ], + "role_arn": "arn:aws:iam::205216182560:role/LabRole", + "s3_backup_configuration": [], + "s3_backup_mode": "Disabled" + } + ], + "http_endpoint_configuration": [], + "id": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687", + "kinesis_source_configuration": [ + { + "kinesis_stream_arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687", + "role_arn": "arn:aws:iam::205216182560:role/LabRole" + } + ], + "msk_source_configuration": [], + "name": "firehose-205216182560-agb-s1201687", + "opensearch_configuration": [], + "opensearchserverless_configuration": [], + "redshift_configuration": [], + "server_side_encryption": [ + { + "enabled": false, + "key_arn": "", + "key_type": "AWS_OWNED_CMK" + } + ], + "snowflake_configuration": [], + "splunk_configuration": [], + "tags": null, + "tags_all": {}, + "timeouts": null, + "version_id": "1" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "aws_kinesis_stream.cryptostock_stream", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_kinesis_stream", + "name": "cryptostock_stream", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687", + "encryption_type": "NONE", + "enforce_consumer_deletion": true, + "id": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687", + "kms_key_id": "", + "name": "cryptostock-205216182560-agb-s1201687", + "retention_period": 24, + "shard_count": 1, + "shard_level_metrics": [ + "IncomingBytes", + "IncomingRecords", + "OutgoingBytes", + "OutgoingRecords" + ], + "stream_mode_details": [ + { + "stream_mode": "PROVISIONED" + } + ], + "tags": { + "Environment": "DEV", + "Owner": "Agnieszka Gąbka-Buszek", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Owner": "Agnieszka Gąbka-Buszek", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_function", + "name": "etl_post_processing", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "architectures": [ + "x86_64" + ], + "arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687", + "code_signing_config_arn": "", + "dead_letter_config": [], + "description": "", + "environment": [], + "ephemeral_storage": [ + { + "size": 512 + } + ], + "file_system_config": [], + "filename": "../lambda/lambda_definition.zip", + "function_name": "etl-post-processing-205216182560-agb-s1201687", + "handler": "lambda_definition.etl_function", + "id": "etl-post-processing-205216182560-agb-s1201687", + "image_config": [], + "image_uri": "", + "invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687/invocations", + "kms_key_arn": "", + "last_modified": "2024-05-30T09:19:35.178+0000", + "layers": [ + "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10" + ], + "logging_config": [ + { + "application_log_level": "", + "log_format": "Text", + "log_group": "/aws/lambda/etl-post-processing-205216182560-agb-s1201687", + "system_log_level": "" + } + ], + "memory_size": 512, + "package_type": "Zip", + "publish": false, + "qualified_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST", + "qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST/invocations", + "replace_security_groups_on_destroy": null, + "replacement_security_group_ids": null, + "reserved_concurrent_executions": -1, + "role": "arn:aws:iam::205216182560:role/LabRole", + "runtime": "python3.8", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "snap_start": [], + "source_code_hash": "rlA5g+hXywn0cpXGqkL9RY3YPioioHFCQQ8jp5Ssjvs=", + "source_code_size": 985, + "tags": null, + "tags_all": {}, + "timeout": 300, + "timeouts": null, + "tracing_config": [ + { + "mode": "PassThrough" + } + ], + "version": "$LATEST", + "vpc_config": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", + "dependencies": [ + "aws_lambda_layer_version.aws_wrangler" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_layer_version", + "name": "aws_wrangler", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10", + "compatible_architectures": null, + "compatible_runtimes": [ + "python3.8" + ], + "created_date": "2024-05-30T09:19:34.959+0000", + "description": "", + "filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip", + "id": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10", + "layer_arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687", + "layer_name": "aws_wrangler_205216182560_agb_s1201687", + "license_info": "", + "s3_bucket": null, + "s3_key": null, + "s3_object_version": null, + "signing_job_arn": "", + "signing_profile_version_arn": "", + "skip_destroy": false, + "source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=", + "source_code_size": 43879070, + "version": "10" + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_lambda_permission", + "name": "allow_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "action": "lambda:InvokeFunction", + "event_source_token": null, + "function_name": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687", + "function_url_auth_type": null, + "id": "AllowExecutionFromS3Bucket", + "principal": "s3.amazonaws.com", + "principal_org_id": null, + "qualifier": "", + "source_account": null, + "source_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687", + "statement_id": "AllowExecutionFromS3Bucket", + "statement_id_prefix": "" + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_lambda_function.etl_post_processing", + "aws_lambda_layer_version.aws_wrangler", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "athena_results", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::athena-results-205216182560-agb-s1201687", + "bucket": "athena-results-205216182560-agb-s1201687", + "bucket_domain_name": "athena-results-205216182560-agb-s1201687.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "athena-results-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "athena-results-205216182560-agb-s1201687", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "environment": "DEV", + "owner": "Agnieszka Gąbka-Buszek", + "purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "environment": "DEV", + "owner": "Agnieszka Gąbka-Buszek", + "purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "processed_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::datalake-processed-205216182560-agb-s1201687", + "bucket": "datalake-processed-205216182560-agb-s1201687", + "bucket_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "datalake-processed-205216182560-agb-s1201687", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "raw_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687", + "bucket": "datalake-raw-205216182560-agb-s1201687", + "bucket_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "datalake-raw-205216182560-agb-s1201687", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_lifecycle_configuration", + "name": "athena_results_lifecycle", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "athena-results-205216182560-agb-s1201687", + "expected_bucket_owner": "", + "id": "athena-results-205216182560-agb-s1201687", + "rule": [ + { + "abort_incomplete_multipart_upload": [], + "expiration": [ + { + "date": "", + "days": 1, + "expired_object_delete_marker": false + } + ], + "filter": [ + { + "and": [], + "object_size_greater_than": "", + "object_size_less_than": "", + "prefix": "", + "tag": [] + } + ], + "id": "standard-expiration", + "noncurrent_version_expiration": [], + "noncurrent_version_transition": [], + "prefix": "", + "status": "Enabled", + "transition": [] + } + ], + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=", + "dependencies": [ + "aws_s3_bucket.athena_results" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket_notification", + "name": "trigger_etl_lambda", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "bucket": "datalake-raw-205216182560-agb-s1201687", + "eventbridge": false, + "id": "datalake-raw-205216182560-agb-s1201687", + "lambda_function": [ + { + "events": [ + "s3:ObjectCreated:*" + ], + "filter_prefix": "raw-zone/", + "filter_suffix": "", + "id": "tf-s3-lambda-20240530091945493300000001", + "lambda_function_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687" + } + ], + "queue": [], + "topic": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_lambda_function.etl_post_processing", + "aws_lambda_layer_version.aws_wrangler", + "aws_lambda_permission.allow_bucket", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_ssm_parameter", + "name": "s3_processed", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "allowed_pattern": "", + "arn": "arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name", + "data_type": "text", + "description": "", + "id": "s3_processed_bucket_name", + "insecure_value": null, + "key_id": "", + "name": "s3_processed_bucket_name", + "overwrite": null, + "tags": null, + "tags_all": {}, + "tier": "Standard", + "type": "String", + "value": "datalake-processed-205216182560-agb-s1201687", + "version": 1 + }, + "sensitive_attributes": [ + [ + { + "type": "get_attr", + "value": "value" + } + ] + ], + "private": "bnVsbA==", + "dependencies": [ + "aws_s3_bucket.processed_bucket" + ] + } + ] + } + ], "check_results": null } diff --git a/labs/terraform/terraform.tfstate.backup b/labs/terraform/terraform.tfstate.backup index 5cf35cd..d7c86f5 100644 --- a/labs/terraform/terraform.tfstate.backup +++ b/labs/terraform/terraform.tfstate.backup @@ -1,896 +1,9 @@ { "version": 4, "terraform_version": "1.8.1", - "serial": 272, + "serial": 323, "lineage": "3ca48c2b-bb88-35e7-8a35-5ef0e08daaff", "outputs": {}, - "resources": [ - { - "mode": "managed", - "type": "aws_athena_workgroup", - "name": "athena_workgroup", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:athena:us-east-1:205216182560:workgroup/development", - "configuration": [ - { - "bytes_scanned_cutoff_per_query": 0, - "enforce_workgroup_configuration": true, - "engine_version": [ - { - "effective_engine_version": "Athena engine version 3", - "selected_engine_version": "AUTO" - } - ], - "execution_role": "", - "publish_cloudwatch_metrics_enabled": true, - "requester_pays_enabled": false, - "result_configuration": [ - { - "acl_configuration": [], - "encryption_configuration": [], - "expected_bucket_owner": "", - "output_location": "s3://athena-results-205216182560-agb-s1201687/output/" - } - ] - } - ], - "description": "", - "force_destroy": true, - "id": "development", - "name": "development", - "state": "ENABLED", - "tags": {}, - "tags_all": {} - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_s3_bucket.athena_results" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_glue_catalog_database", - "name": "datalake_db_processed_zone", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_processed_205216182560_agb_s1201687", - "catalog_id": "205216182560", - "create_table_default_permission": [ - { - "permissions": [ - "ALL" - ], - "principal": [ - { - "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" - } - ] - } - ], - "description": "", - "federated_database": [], - "id": "205216182560:datalake_processed_205216182560_agb_s1201687", - "location_uri": "", - "name": "datalake_processed_205216182560_agb_s1201687", - "parameters": {}, - "tags": {}, - "tags_all": {}, - "target_database": [] - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - }, - { - "mode": "managed", - "type": "aws_glue_catalog_database", - "name": "datalake_db_raw_zone", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_raw_205216182560_agb_s1201687", - "catalog_id": "205216182560", - "create_table_default_permission": [ - { - "permissions": [ - "ALL" - ], - "principal": [ - { - "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" - } - ] - } - ], - "description": "", - "federated_database": [], - "id": "205216182560:datalake_raw_205216182560_agb_s1201687", - "location_uri": "", - "name": "datalake_raw_205216182560_agb_s1201687", - "parameters": {}, - "tags": {}, - "tags_all": {}, - "target_database": [] - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - }, - { - "mode": "managed", - "type": "aws_glue_crawler", - "name": "glue_crawler_raw_zone", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:glue:us-east-1:205216182560:crawler/gc-raw-205216182560-agb-s1201687", - "catalog_target": [], - "classifiers": [], - "configuration": "", - "database_name": "datalake_raw_205216182560_agb_s1201687", - "delta_target": [], - "description": "", - "dynamodb_target": [], - "hudi_target": [], - "iceberg_target": [], - "id": "gc-raw-205216182560-agb-s1201687", - "jdbc_target": [], - "lake_formation_configuration": [ - { - "account_id": "", - "use_lake_formation_credentials": false - } - ], - "lineage_configuration": [ - { - "crawler_lineage_settings": "DISABLE" - } - ], - "mongodb_target": [], - "name": "gc-raw-205216182560-agb-s1201687", - "recrawl_policy": [ - { - "recrawl_behavior": "CRAWL_EVERYTHING" - } - ], - "role": "LabRole", - "s3_target": [ - { - "connection_name": "", - "dlq_event_queue_arn": "", - "event_queue_arn": "", - "exclusions": [], - "path": "s3://datalake-raw-205216182560-agb-s1201687/raw-zone/stockdata/", - "sample_size": 0 - } - ], - "schedule": "", - "schema_change_policy": [ - { - "delete_behavior": "DEPRECATE_IN_DATABASE", - "update_behavior": "UPDATE_IN_DATABASE" - } - ], - "security_configuration": "", - "table_prefix": "crawler_", - "tags": { - "environment": "DEV", - "owner": "Agnieszka Gąbka-Buszek", - "purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "environment": "DEV", - "owner": "Agnieszka Gąbka-Buszek", - "purpose": "UAM Cloud Data Processing" - } - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_glue_catalog_database.datalake_db_raw_zone", - "aws_s3_bucket.raw_bucket" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_iam_policy", - "name": "lambda_policy", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:iam::205216182560:policy/lambda_policy", - "attachment_count": 0, - "description": "", - "id": "arn:aws:iam::205216182560:policy/lambda_policy", - "name": "lambda_policy", - "name_prefix": "", - "path": "/", - "policy": "{\"Statement\":[{\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::datalake-raw-205216182560-agb-s1201687/*\",\"arn:aws:s3:::datalake-processed-205216182560-agb-s1201687/*\"]},{\"Action\":\"ssm:GetParameter\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name\"}],\"Version\":\"2012-10-17\"}", - "policy_id": "ANPAS7R6WOEQGHVSNQS6X", - "tags": {}, - "tags_all": {} - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_s3_bucket.processed_bucket", - "aws_s3_bucket.raw_bucket" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_kinesis_firehose_delivery_stream", - "name": "stock_delivery_stream", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687", - "destination": "extended_s3", - "destination_id": "destinationId-000000000001", - "elasticsearch_configuration": [], - "extended_s3_configuration": [ - { - "bucket_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687", - "buffering_interval": 60, - "buffering_size": 1, - "cloudwatch_logging_options": [ - { - "enabled": false, - "log_group_name": "", - "log_stream_name": "" - } - ], - "compression_format": "UNCOMPRESSED", - "custom_time_zone": "UTC", - "data_format_conversion_configuration": [], - "dynamic_partitioning_configuration": [], - "error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", - "file_extension": "", - "kms_key_arn": "", - "prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", - "processing_configuration": [ - { - "enabled": false, - "processors": [] - } - ], - "role_arn": "arn:aws:iam::205216182560:role/LabRole", - "s3_backup_configuration": [], - "s3_backup_mode": "Disabled" - } - ], - "http_endpoint_configuration": [], - "id": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687", - "kinesis_source_configuration": [ - { - "kinesis_stream_arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687", - "role_arn": "arn:aws:iam::205216182560:role/LabRole" - } - ], - "msk_source_configuration": [], - "name": "firehose-205216182560-agb-s1201687", - "opensearch_configuration": [], - "opensearchserverless_configuration": [], - "redshift_configuration": [], - "server_side_encryption": [ - { - "enabled": false, - "key_arn": "", - "key_type": "AWS_OWNED_CMK" - } - ], - "snowflake_configuration": [], - "splunk_configuration": [], - "tags": {}, - "tags_all": {}, - "timeouts": null, - "version_id": "1" - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", - "dependencies": [ - "aws_kinesis_stream.cryptostock_stream", - "aws_s3_bucket.raw_bucket" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_kinesis_stream", - "name": "cryptostock_stream", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 1, - "attributes": { - "arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687", - "encryption_type": "NONE", - "enforce_consumer_deletion": true, - "id": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687", - "kms_key_id": "", - "name": "cryptostock-205216182560-agb-s1201687", - "retention_period": 24, - "shard_count": 1, - "shard_level_metrics": [ - "IncomingBytes", - "IncomingRecords", - "OutgoingBytes", - "OutgoingRecords" - ], - "stream_mode_details": [ - { - "stream_mode": "PROVISIONED" - } - ], - "tags": { - "Environment": "DEV", - "Owner": "Agnieszka Gąbka-Buszek", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Owner": "Agnieszka Gąbka-Buszek", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" - } - ] - }, - { - "mode": "managed", - "type": "aws_lambda_function", - "name": "etl_post_processing", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "architectures": [ - "x86_64" - ], - "arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687", - "code_signing_config_arn": "", - "dead_letter_config": [], - "description": "", - "environment": [], - "ephemeral_storage": [ - { - "size": 512 - } - ], - "file_system_config": [], - "filename": "../lambda/lambda_definition.zip", - "function_name": "etl-post-processing-205216182560-agb-s1201687", - "handler": "lambda_definition.etl_function", - "id": "etl-post-processing-205216182560-agb-s1201687", - "image_config": [], - "image_uri": "", - "invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687/invocations", - "kms_key_arn": "", - "last_modified": "2024-05-27T10:23:37.000+0000", - "layers": [ - "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7" - ], - "logging_config": [ - { - "application_log_level": "", - "log_format": "Text", - "log_group": "/aws/lambda/etl-post-processing-205216182560-agb-s1201687", - "system_log_level": "" - } - ], - "memory_size": 512, - "package_type": "Zip", - "publish": false, - "qualified_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST", - "qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST/invocations", - "replace_security_groups_on_destroy": null, - "replacement_security_group_ids": null, - "reserved_concurrent_executions": -1, - "role": "arn:aws:iam::205216182560:role/LabRole", - "runtime": "python3.8", - "s3_bucket": null, - "s3_key": null, - "s3_object_version": null, - "signing_job_arn": "", - "signing_profile_version_arn": "", - "skip_destroy": false, - "snap_start": [], - "source_code_hash": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=", - "source_code_size": 884, - "tags": {}, - "tags_all": {}, - "timeout": 300, - "timeouts": null, - "tracing_config": [ - { - "mode": "PassThrough" - } - ], - "version": "$LATEST", - "vpc_config": [] - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19", - "dependencies": [ - "aws_lambda_layer_version.aws_wrangler" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_lambda_layer_version", - "name": "aws_wrangler", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7", - "compatible_architectures": [], - "compatible_runtimes": [ - "python3.8" - ], - "created_date": "2024-05-27T08:03:57.293+0000", - "description": "", - "filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip", - "id": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7", - "layer_arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687", - "layer_name": "aws_wrangler_205216182560_agb_s1201687", - "license_info": "", - "s3_bucket": null, - "s3_key": null, - "s3_object_version": null, - "signing_job_arn": "", - "signing_profile_version_arn": "", - "skip_destroy": false, - "source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=", - "source_code_size": 43879070, - "version": "7" - }, - "sensitive_attributes": [], - "private": "bnVsbA==" - } - ] - }, - { - "mode": "managed", - "type": "aws_lambda_permission", - "name": "allow_bucket", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "action": "lambda:InvokeFunction", - "event_source_token": null, - "function_name": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687", - "function_url_auth_type": null, - "id": "AllowExecutionFromS3Bucket", - "principal": "s3.amazonaws.com", - "principal_org_id": null, - "qualifier": "", - "source_account": null, - "source_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687", - "statement_id": "AllowExecutionFromS3Bucket", - "statement_id_prefix": "" - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_lambda_function.etl_post_processing", - "aws_lambda_layer_version.aws_wrangler", - "aws_s3_bucket.raw_bucket" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket", - "name": "athena_results", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "acceleration_status": "", - "acl": null, - "arn": "arn:aws:s3:::athena-results-205216182560-agb-s1201687", - "bucket": "athena-results-205216182560-agb-s1201687", - "bucket_domain_name": "athena-results-205216182560-agb-s1201687.s3.amazonaws.com", - "bucket_prefix": "", - "bucket_regional_domain_name": "athena-results-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com", - "cors_rule": [], - "force_destroy": true, - "grant": [ - { - "id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e", - "permissions": [ - "FULL_CONTROL" - ], - "type": "CanonicalUser", - "uri": "" - } - ], - "hosted_zone_id": "Z3AQBSTGFYJSTF", - "id": "athena-results-205216182560-agb-s1201687", - "lifecycle_rule": [ - { - "abort_incomplete_multipart_upload_days": 0, - "enabled": true, - "expiration": [ - { - "date": "", - "days": 1, - "expired_object_delete_marker": false - } - ], - "id": "standard-expiration", - "noncurrent_version_expiration": [], - "noncurrent_version_transition": [], - "prefix": "", - "tags": {}, - "transition": [] - } - ], - "logging": [], - "object_lock_configuration": [], - "object_lock_enabled": false, - "policy": "", - "region": "us-east-1", - "replication_configuration": [], - "request_payer": "BucketOwner", - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": "", - "sse_algorithm": "AES256" - } - ], - "bucket_key_enabled": false - } - ] - } - ], - "tags": { - "environment": "DEV", - "owner": "Agnieszka Gąbka-Buszek", - "purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "environment": "DEV", - "owner": "Agnieszka Gąbka-Buszek", - "purpose": "UAM Cloud Data Processing" - }, - "timeouts": null, - "versioning": [ - { - "enabled": false, - "mfa_delete": false - } - ], - "website": [], - "website_domain": null, - "website_endpoint": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket", - "name": "processed_bucket", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "acceleration_status": "", - "acl": null, - "arn": "arn:aws:s3:::datalake-processed-205216182560-agb-s1201687", - "bucket": "datalake-processed-205216182560-agb-s1201687", - "bucket_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.amazonaws.com", - "bucket_prefix": "", - "bucket_regional_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com", - "cors_rule": [], - "force_destroy": true, - "grant": [ - { - "id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e", - "permissions": [ - "FULL_CONTROL" - ], - "type": "CanonicalUser", - "uri": "" - } - ], - "hosted_zone_id": "Z3AQBSTGFYJSTF", - "id": "datalake-processed-205216182560-agb-s1201687", - "lifecycle_rule": [], - "logging": [], - "object_lock_configuration": [], - "object_lock_enabled": false, - "policy": "", - "region": "us-east-1", - "replication_configuration": [], - "request_payer": "BucketOwner", - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": "", - "sse_algorithm": "AES256" - } - ], - "bucket_key_enabled": false - } - ] - } - ], - "tags": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null, - "versioning": [ - { - "enabled": false, - "mfa_delete": false - } - ], - "website": [], - "website_domain": null, - "website_endpoint": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket", - "name": "raw_bucket", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "acceleration_status": "", - "acl": null, - "arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687", - "bucket": "datalake-raw-205216182560-agb-s1201687", - "bucket_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.amazonaws.com", - "bucket_prefix": "", - "bucket_regional_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com", - "cors_rule": [], - "force_destroy": true, - "grant": [ - { - "id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e", - "permissions": [ - "FULL_CONTROL" - ], - "type": "CanonicalUser", - "uri": "" - } - ], - "hosted_zone_id": "Z3AQBSTGFYJSTF", - "id": "datalake-raw-205216182560-agb-s1201687", - "lifecycle_rule": [], - "logging": [], - "object_lock_configuration": [], - "object_lock_enabled": false, - "policy": "", - "region": "us-east-1", - "replication_configuration": [], - "request_payer": "BucketOwner", - "server_side_encryption_configuration": [ - { - "rule": [ - { - "apply_server_side_encryption_by_default": [ - { - "kms_master_key_id": "", - "sse_algorithm": "AES256" - } - ], - "bucket_key_enabled": false - } - ] - } - ], - "tags": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "tags_all": { - "Environment": "DEV", - "Purpose": "UAM Cloud Data Processing" - }, - "timeouts": null, - "versioning": [ - { - "enabled": false, - "mfa_delete": false - } - ], - "website": [], - "website_domain": null, - "website_endpoint": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket_lifecycle_configuration", - "name": "athena_results_lifecycle", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "bucket": "athena-results-205216182560-agb-s1201687", - "expected_bucket_owner": "", - "id": "athena-results-205216182560-agb-s1201687", - "rule": [ - { - "abort_incomplete_multipart_upload": [], - "expiration": [ - { - "date": "", - "days": 1, - "expired_object_delete_marker": false - } - ], - "filter": [ - { - "and": [], - "object_size_greater_than": "", - "object_size_less_than": "", - "prefix": "", - "tag": [] - } - ], - "id": "standard-expiration", - "noncurrent_version_expiration": [], - "noncurrent_version_transition": [], - "prefix": "", - "status": "Enabled", - "transition": [] - } - ], - "timeouts": null - }, - "sensitive_attributes": [], - "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=", - "dependencies": [ - "aws_s3_bucket.athena_results" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_s3_bucket_notification", - "name": "trigger_etl_lambda", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "bucket": "datalake-raw-205216182560-agb-s1201687", - "eventbridge": false, - "id": "datalake-raw-205216182560-agb-s1201687", - "lambda_function": [ - { - "events": [ - "s3:ObjectCreated:*" - ], - "filter_prefix": "raw-zone/", - "filter_suffix": "", - "id": "tf-s3-lambda-20240527080407879000000001", - "lambda_function_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687" - } - ], - "queue": [], - "topic": [] - }, - "sensitive_attributes": [], - "private": "bnVsbA==", - "dependencies": [ - "aws_lambda_function.etl_post_processing", - "aws_lambda_layer_version.aws_wrangler", - "aws_lambda_permission.allow_bucket", - "aws_s3_bucket.raw_bucket" - ] - } - ] - }, - { - "mode": "managed", - "type": "aws_ssm_parameter", - "name": "s3_processed", - "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", - "instances": [ - { - "schema_version": 0, - "attributes": { - "allowed_pattern": "", - "arn": "arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name", - "data_type": "text", - "description": "", - "id": "s3_processed_bucket_name", - "insecure_value": null, - "key_id": "", - "name": "s3_processed_bucket_name", - "overwrite": null, - "tags": {}, - "tags_all": {}, - "tier": "Standard", - "type": "String", - "value": "datalake-processed-205216182560-agb-s1201687", - "version": 1 - }, - "sensitive_attributes": [ - [ - { - "type": "get_attr", - "value": "value" - } - ] - ], - "private": "bnVsbA==", - "dependencies": [ - "aws_s3_bucket.processed_bucket" - ] - } - ] - } - ], + "resources": [], "check_results": null }