Dodano nowe pliki i zmiany w kodzie Lambda
This commit is contained in:
EC2 Default User
parent
b6145320e9
commit
bc1f2ad333
@ -9,16 +9,19 @@ def etl_function(event, context):
|
||||
processed_zone_prefix = "processed-zone"
|
||||
|
||||
record = event["Records"][0]
|
||||
bucket = client_ssm.get_parameter(Name = 's3_processed_bucket_name')['Parameter']['Value']
|
||||
raw_bucket = record["s3"]["bucket"]["name"]
|
||||
key = urllib.parse.unquote(record["s3"]["object"]["key"])
|
||||
event_prefix = key.split('/')[1]
|
||||
full_src_path = 's3://{bucket}/{key}'.format(bucket=bucket, key=key)
|
||||
full_src_path = f's3://{raw_bucket}/{key}'
|
||||
|
||||
# Pobierz nazwę bucketu przetworzonych danych z SSM
|
||||
processed_bucket = client_ssm.get_parameter(Name='s3_processed_bucket_name')['Parameter']['Value']
|
||||
|
||||
print(f'Processing key = {full_src_path}')
|
||||
df = wr.s3.read_json(path=full_src_path, lines=True)
|
||||
|
||||
filename = key.split('/')[-1][-36:]
|
||||
dest_prefix = f"s3://{bucket}/{processed_zone_prefix}/{event_prefix}"
|
||||
dest_prefix = f"s3://{processed_bucket}/{processed_zone_prefix}/{event_prefix}"
|
||||
|
||||
df['transaction_date'] = pd.to_datetime(df['transaction_ts'], unit='s')
|
||||
df['year'] = df['transaction_date'].dt.year
|
||||
@ -46,11 +49,4 @@ def etl_function(event, context):
|
||||
'key': key,
|
||||
'statusCode': 200,
|
||||
'new_keys': new_keys
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
event = ""
|
||||
context = ""
|
||||
|
||||
response = etl_function(event, context)
|
||||
}
|
||||
Binary file not shown.
@ -1,9 +1,878 @@
|
||||
{
|
||||
"version": 4,
|
||||
"terraform_version": "1.8.1",
|
||||
"serial": 289,
|
||||
"serial": 340,
|
||||
"lineage": "3ca48c2b-bb88-35e7-8a35-5ef0e08daaff",
|
||||
"outputs": {},
|
||||
"resources": [],
|
||||
"resources": [
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_athena_workgroup",
|
||||
"name": "athena_workgroup",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:athena:us-east-1:205216182560:workgroup/development",
|
||||
"configuration": [
|
||||
{
|
||||
"bytes_scanned_cutoff_per_query": 0,
|
||||
"enforce_workgroup_configuration": true,
|
||||
"engine_version": [
|
||||
{
|
||||
"effective_engine_version": "Athena engine version 3",
|
||||
"selected_engine_version": "AUTO"
|
||||
}
|
||||
],
|
||||
"execution_role": "",
|
||||
"publish_cloudwatch_metrics_enabled": true,
|
||||
"requester_pays_enabled": false,
|
||||
"result_configuration": [
|
||||
{
|
||||
"acl_configuration": [],
|
||||
"encryption_configuration": [],
|
||||
"expected_bucket_owner": "",
|
||||
"output_location": "s3://athena-results-205216182560-agb-s1201687/output/"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"force_destroy": true,
|
||||
"id": "development",
|
||||
"name": "development",
|
||||
"state": "ENABLED",
|
||||
"tags": null,
|
||||
"tags_all": {}
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_s3_bucket.athena_results"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_glue_catalog_database",
|
||||
"name": "datalake_db_processed_zone",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_processed_205216182560_agb_s1201687",
|
||||
"catalog_id": "205216182560",
|
||||
"create_table_default_permission": [
|
||||
{
|
||||
"permissions": [
|
||||
"ALL"
|
||||
],
|
||||
"principal": [
|
||||
{
|
||||
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"federated_database": [],
|
||||
"id": "205216182560:datalake_processed_205216182560_agb_s1201687",
|
||||
"location_uri": "",
|
||||
"name": "datalake_processed_205216182560_agb_s1201687",
|
||||
"parameters": null,
|
||||
"tags": null,
|
||||
"tags_all": {},
|
||||
"target_database": []
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_glue_catalog_database",
|
||||
"name": "datalake_db_raw_zone",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_raw_205216182560_agb_s1201687",
|
||||
"catalog_id": "205216182560",
|
||||
"create_table_default_permission": [
|
||||
{
|
||||
"permissions": [
|
||||
"ALL"
|
||||
],
|
||||
"principal": [
|
||||
{
|
||||
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"federated_database": [],
|
||||
"id": "205216182560:datalake_raw_205216182560_agb_s1201687",
|
||||
"location_uri": "",
|
||||
"name": "datalake_raw_205216182560_agb_s1201687",
|
||||
"parameters": null,
|
||||
"tags": null,
|
||||
"tags_all": {},
|
||||
"target_database": []
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_glue_crawler",
|
||||
"name": "glue_crawler_raw_zone",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:glue:us-east-1:205216182560:crawler/gc-raw-205216182560-agb-s1201687",
|
||||
"catalog_target": [],
|
||||
"classifiers": null,
|
||||
"configuration": "",
|
||||
"database_name": "datalake_raw_205216182560_agb_s1201687",
|
||||
"delta_target": [],
|
||||
"description": "",
|
||||
"dynamodb_target": [],
|
||||
"hudi_target": [],
|
||||
"iceberg_target": [],
|
||||
"id": "gc-raw-205216182560-agb-s1201687",
|
||||
"jdbc_target": [],
|
||||
"lake_formation_configuration": [
|
||||
{
|
||||
"account_id": "",
|
||||
"use_lake_formation_credentials": false
|
||||
}
|
||||
],
|
||||
"lineage_configuration": [
|
||||
{
|
||||
"crawler_lineage_settings": "DISABLE"
|
||||
}
|
||||
],
|
||||
"mongodb_target": [],
|
||||
"name": "gc-raw-205216182560-agb-s1201687",
|
||||
"recrawl_policy": [
|
||||
{
|
||||
"recrawl_behavior": "CRAWL_EVERYTHING"
|
||||
}
|
||||
],
|
||||
"role": "LabRole",
|
||||
"s3_target": [
|
||||
{
|
||||
"connection_name": "",
|
||||
"dlq_event_queue_arn": "",
|
||||
"event_queue_arn": "",
|
||||
"exclusions": null,
|
||||
"path": "s3://datalake-raw-205216182560-agb-s1201687/raw-zone/stockdata/",
|
||||
"sample_size": 0
|
||||
}
|
||||
],
|
||||
"schedule": "",
|
||||
"schema_change_policy": [
|
||||
{
|
||||
"delete_behavior": "DEPRECATE_IN_DATABASE",
|
||||
"update_behavior": "UPDATE_IN_DATABASE"
|
||||
}
|
||||
],
|
||||
"security_configuration": "",
|
||||
"table_prefix": "crawler_",
|
||||
"tags": {
|
||||
"environment": "DEV",
|
||||
"owner": "Agnieszka Gąbka-Buszek",
|
||||
"purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"environment": "DEV",
|
||||
"owner": "Agnieszka Gąbka-Buszek",
|
||||
"purpose": "UAM Cloud Data Processing"
|
||||
}
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_glue_catalog_database.datalake_db_raw_zone",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_iam_policy",
|
||||
"name": "lambda_policy",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:iam::205216182560:policy/lambda_policy",
|
||||
"attachment_count": 0,
|
||||
"description": "",
|
||||
"id": "arn:aws:iam::205216182560:policy/lambda_policy",
|
||||
"name": "lambda_policy",
|
||||
"name_prefix": "",
|
||||
"path": "/",
|
||||
"policy": "{\"Statement\":[{\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::datalake-raw-205216182560-agb-s1201687/*\",\"arn:aws:s3:::datalake-processed-205216182560-agb-s1201687/*\"]},{\"Action\":\"ssm:GetParameter\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name\"}],\"Version\":\"2012-10-17\"}",
|
||||
"policy_id": "ANPAS7R6WOEQMJ2ODN7S5",
|
||||
"tags": null,
|
||||
"tags_all": {}
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_s3_bucket.processed_bucket",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_kinesis_firehose_delivery_stream",
|
||||
"name": "stock_delivery_stream",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687",
|
||||
"destination": "extended_s3",
|
||||
"destination_id": "destinationId-000000000001",
|
||||
"elasticsearch_configuration": [],
|
||||
"extended_s3_configuration": [
|
||||
{
|
||||
"bucket_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
|
||||
"buffering_interval": 60,
|
||||
"buffering_size": 1,
|
||||
"cloudwatch_logging_options": [
|
||||
{
|
||||
"enabled": false,
|
||||
"log_group_name": "",
|
||||
"log_stream_name": ""
|
||||
}
|
||||
],
|
||||
"compression_format": "UNCOMPRESSED",
|
||||
"custom_time_zone": "UTC",
|
||||
"data_format_conversion_configuration": [],
|
||||
"dynamic_partitioning_configuration": [],
|
||||
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
||||
"file_extension": "",
|
||||
"kms_key_arn": "",
|
||||
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
||||
"processing_configuration": [
|
||||
{
|
||||
"enabled": false,
|
||||
"processors": []
|
||||
}
|
||||
],
|
||||
"role_arn": "arn:aws:iam::205216182560:role/LabRole",
|
||||
"s3_backup_configuration": [],
|
||||
"s3_backup_mode": "Disabled"
|
||||
}
|
||||
],
|
||||
"http_endpoint_configuration": [],
|
||||
"id": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687",
|
||||
"kinesis_source_configuration": [
|
||||
{
|
||||
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
|
||||
"role_arn": "arn:aws:iam::205216182560:role/LabRole"
|
||||
}
|
||||
],
|
||||
"msk_source_configuration": [],
|
||||
"name": "firehose-205216182560-agb-s1201687",
|
||||
"opensearch_configuration": [],
|
||||
"opensearchserverless_configuration": [],
|
||||
"redshift_configuration": [],
|
||||
"server_side_encryption": [
|
||||
{
|
||||
"enabled": false,
|
||||
"key_arn": "",
|
||||
"key_type": "AWS_OWNED_CMK"
|
||||
}
|
||||
],
|
||||
"snowflake_configuration": [],
|
||||
"splunk_configuration": [],
|
||||
"tags": null,
|
||||
"tags_all": {},
|
||||
"timeouts": null,
|
||||
"version_id": "1"
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
|
||||
"dependencies": [
|
||||
"aws_kinesis_stream.cryptostock_stream",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_kinesis_stream",
|
||||
"name": "cryptostock_stream",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
|
||||
"encryption_type": "NONE",
|
||||
"enforce_consumer_deletion": true,
|
||||
"id": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
|
||||
"kms_key_id": "",
|
||||
"name": "cryptostock-205216182560-agb-s1201687",
|
||||
"retention_period": 24,
|
||||
"shard_count": 1,
|
||||
"shard_level_metrics": [
|
||||
"IncomingBytes",
|
||||
"IncomingRecords",
|
||||
"OutgoingBytes",
|
||||
"OutgoingRecords"
|
||||
],
|
||||
"stream_mode_details": [
|
||||
{
|
||||
"stream_mode": "PROVISIONED"
|
||||
}
|
||||
],
|
||||
"tags": {
|
||||
"Environment": "DEV",
|
||||
"Owner": "Agnieszka Gąbka-Buszek",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"Environment": "DEV",
|
||||
"Owner": "Agnieszka Gąbka-Buszek",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"timeouts": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_lambda_function",
|
||||
"name": "etl_post_processing",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"architectures": [
|
||||
"x86_64"
|
||||
],
|
||||
"arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687",
|
||||
"code_signing_config_arn": "",
|
||||
"dead_letter_config": [],
|
||||
"description": "",
|
||||
"environment": [],
|
||||
"ephemeral_storage": [
|
||||
{
|
||||
"size": 512
|
||||
}
|
||||
],
|
||||
"file_system_config": [],
|
||||
"filename": "../lambda/lambda_definition.zip",
|
||||
"function_name": "etl-post-processing-205216182560-agb-s1201687",
|
||||
"handler": "lambda_definition.etl_function",
|
||||
"id": "etl-post-processing-205216182560-agb-s1201687",
|
||||
"image_config": [],
|
||||
"image_uri": "",
|
||||
"invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687/invocations",
|
||||
"kms_key_arn": "",
|
||||
"last_modified": "2024-05-30T09:19:35.178+0000",
|
||||
"layers": [
|
||||
"arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10"
|
||||
],
|
||||
"logging_config": [
|
||||
{
|
||||
"application_log_level": "",
|
||||
"log_format": "Text",
|
||||
"log_group": "/aws/lambda/etl-post-processing-205216182560-agb-s1201687",
|
||||
"system_log_level": ""
|
||||
}
|
||||
],
|
||||
"memory_size": 512,
|
||||
"package_type": "Zip",
|
||||
"publish": false,
|
||||
"qualified_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST",
|
||||
"qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST/invocations",
|
||||
"replace_security_groups_on_destroy": null,
|
||||
"replacement_security_group_ids": null,
|
||||
"reserved_concurrent_executions": -1,
|
||||
"role": "arn:aws:iam::205216182560:role/LabRole",
|
||||
"runtime": "python3.8",
|
||||
"s3_bucket": null,
|
||||
"s3_key": null,
|
||||
"s3_object_version": null,
|
||||
"signing_job_arn": "",
|
||||
"signing_profile_version_arn": "",
|
||||
"skip_destroy": false,
|
||||
"snap_start": [],
|
||||
"source_code_hash": "rlA5g+hXywn0cpXGqkL9RY3YPioioHFCQQ8jp5Ssjvs=",
|
||||
"source_code_size": 985,
|
||||
"tags": null,
|
||||
"tags_all": {},
|
||||
"timeout": 300,
|
||||
"timeouts": null,
|
||||
"tracing_config": [
|
||||
{
|
||||
"mode": "PassThrough"
|
||||
}
|
||||
],
|
||||
"version": "$LATEST",
|
||||
"vpc_config": []
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19",
|
||||
"dependencies": [
|
||||
"aws_lambda_layer_version.aws_wrangler"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_lambda_layer_version",
|
||||
"name": "aws_wrangler",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10",
|
||||
"compatible_architectures": null,
|
||||
"compatible_runtimes": [
|
||||
"python3.8"
|
||||
],
|
||||
"created_date": "2024-05-30T09:19:34.959+0000",
|
||||
"description": "",
|
||||
"filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip",
|
||||
"id": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:10",
|
||||
"layer_arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687",
|
||||
"layer_name": "aws_wrangler_205216182560_agb_s1201687",
|
||||
"license_info": "",
|
||||
"s3_bucket": null,
|
||||
"s3_key": null,
|
||||
"s3_object_version": null,
|
||||
"signing_job_arn": "",
|
||||
"signing_profile_version_arn": "",
|
||||
"skip_destroy": false,
|
||||
"source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=",
|
||||
"source_code_size": 43879070,
|
||||
"version": "10"
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_lambda_permission",
|
||||
"name": "allow_bucket",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"action": "lambda:InvokeFunction",
|
||||
"event_source_token": null,
|
||||
"function_name": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687",
|
||||
"function_url_auth_type": null,
|
||||
"id": "AllowExecutionFromS3Bucket",
|
||||
"principal": "s3.amazonaws.com",
|
||||
"principal_org_id": null,
|
||||
"qualifier": "",
|
||||
"source_account": null,
|
||||
"source_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
|
||||
"statement_id": "AllowExecutionFromS3Bucket",
|
||||
"statement_id_prefix": ""
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_lambda_function.etl_post_processing",
|
||||
"aws_lambda_layer_version.aws_wrangler",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket",
|
||||
"name": "athena_results",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"acceleration_status": "",
|
||||
"acl": null,
|
||||
"arn": "arn:aws:s3:::athena-results-205216182560-agb-s1201687",
|
||||
"bucket": "athena-results-205216182560-agb-s1201687",
|
||||
"bucket_domain_name": "athena-results-205216182560-agb-s1201687.s3.amazonaws.com",
|
||||
"bucket_prefix": "",
|
||||
"bucket_regional_domain_name": "athena-results-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
|
||||
"cors_rule": [],
|
||||
"force_destroy": true,
|
||||
"grant": [
|
||||
{
|
||||
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
|
||||
"permissions": [
|
||||
"FULL_CONTROL"
|
||||
],
|
||||
"type": "CanonicalUser",
|
||||
"uri": ""
|
||||
}
|
||||
],
|
||||
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||
"id": "athena-results-205216182560-agb-s1201687",
|
||||
"lifecycle_rule": [],
|
||||
"logging": [],
|
||||
"object_lock_configuration": [],
|
||||
"object_lock_enabled": false,
|
||||
"policy": "",
|
||||
"region": "us-east-1",
|
||||
"replication_configuration": [],
|
||||
"request_payer": "BucketOwner",
|
||||
"server_side_encryption_configuration": [
|
||||
{
|
||||
"rule": [
|
||||
{
|
||||
"apply_server_side_encryption_by_default": [
|
||||
{
|
||||
"kms_master_key_id": "",
|
||||
"sse_algorithm": "AES256"
|
||||
}
|
||||
],
|
||||
"bucket_key_enabled": false
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": {
|
||||
"environment": "DEV",
|
||||
"owner": "Agnieszka Gąbka-Buszek",
|
||||
"purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"environment": "DEV",
|
||||
"owner": "Agnieszka Gąbka-Buszek",
|
||||
"purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"timeouts": null,
|
||||
"versioning": [
|
||||
{
|
||||
"enabled": false,
|
||||
"mfa_delete": false
|
||||
}
|
||||
],
|
||||
"website": [],
|
||||
"website_domain": null,
|
||||
"website_endpoint": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket",
|
||||
"name": "processed_bucket",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"acceleration_status": "",
|
||||
"acl": null,
|
||||
"arn": "arn:aws:s3:::datalake-processed-205216182560-agb-s1201687",
|
||||
"bucket": "datalake-processed-205216182560-agb-s1201687",
|
||||
"bucket_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.amazonaws.com",
|
||||
"bucket_prefix": "",
|
||||
"bucket_regional_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
|
||||
"cors_rule": [],
|
||||
"force_destroy": true,
|
||||
"grant": [
|
||||
{
|
||||
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
|
||||
"permissions": [
|
||||
"FULL_CONTROL"
|
||||
],
|
||||
"type": "CanonicalUser",
|
||||
"uri": ""
|
||||
}
|
||||
],
|
||||
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||
"id": "datalake-processed-205216182560-agb-s1201687",
|
||||
"lifecycle_rule": [],
|
||||
"logging": [],
|
||||
"object_lock_configuration": [],
|
||||
"object_lock_enabled": false,
|
||||
"policy": "",
|
||||
"region": "us-east-1",
|
||||
"replication_configuration": [],
|
||||
"request_payer": "BucketOwner",
|
||||
"server_side_encryption_configuration": [
|
||||
{
|
||||
"rule": [
|
||||
{
|
||||
"apply_server_side_encryption_by_default": [
|
||||
{
|
||||
"kms_master_key_id": "",
|
||||
"sse_algorithm": "AES256"
|
||||
}
|
||||
],
|
||||
"bucket_key_enabled": false
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": {
|
||||
"Environment": "DEV",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"Environment": "DEV",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"timeouts": null,
|
||||
"versioning": [
|
||||
{
|
||||
"enabled": false,
|
||||
"mfa_delete": false
|
||||
}
|
||||
],
|
||||
"website": [],
|
||||
"website_domain": null,
|
||||
"website_endpoint": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket",
|
||||
"name": "raw_bucket",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"acceleration_status": "",
|
||||
"acl": null,
|
||||
"arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
|
||||
"bucket": "datalake-raw-205216182560-agb-s1201687",
|
||||
"bucket_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.amazonaws.com",
|
||||
"bucket_prefix": "",
|
||||
"bucket_regional_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
|
||||
"cors_rule": [],
|
||||
"force_destroy": true,
|
||||
"grant": [
|
||||
{
|
||||
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
|
||||
"permissions": [
|
||||
"FULL_CONTROL"
|
||||
],
|
||||
"type": "CanonicalUser",
|
||||
"uri": ""
|
||||
}
|
||||
],
|
||||
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||
"id": "datalake-raw-205216182560-agb-s1201687",
|
||||
"lifecycle_rule": [],
|
||||
"logging": [],
|
||||
"object_lock_configuration": [],
|
||||
"object_lock_enabled": false,
|
||||
"policy": "",
|
||||
"region": "us-east-1",
|
||||
"replication_configuration": [],
|
||||
"request_payer": "BucketOwner",
|
||||
"server_side_encryption_configuration": [
|
||||
{
|
||||
"rule": [
|
||||
{
|
||||
"apply_server_side_encryption_by_default": [
|
||||
{
|
||||
"kms_master_key_id": "",
|
||||
"sse_algorithm": "AES256"
|
||||
}
|
||||
],
|
||||
"bucket_key_enabled": false
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": {
|
||||
"Environment": "DEV",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"Environment": "DEV",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"timeouts": null,
|
||||
"versioning": [
|
||||
{
|
||||
"enabled": false,
|
||||
"mfa_delete": false
|
||||
}
|
||||
],
|
||||
"website": [],
|
||||
"website_domain": null,
|
||||
"website_endpoint": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket_lifecycle_configuration",
|
||||
"name": "athena_results_lifecycle",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"bucket": "athena-results-205216182560-agb-s1201687",
|
||||
"expected_bucket_owner": "",
|
||||
"id": "athena-results-205216182560-agb-s1201687",
|
||||
"rule": [
|
||||
{
|
||||
"abort_incomplete_multipart_upload": [],
|
||||
"expiration": [
|
||||
{
|
||||
"date": "",
|
||||
"days": 1,
|
||||
"expired_object_delete_marker": false
|
||||
}
|
||||
],
|
||||
"filter": [
|
||||
{
|
||||
"and": [],
|
||||
"object_size_greater_than": "",
|
||||
"object_size_less_than": "",
|
||||
"prefix": "",
|
||||
"tag": []
|
||||
}
|
||||
],
|
||||
"id": "standard-expiration",
|
||||
"noncurrent_version_expiration": [],
|
||||
"noncurrent_version_transition": [],
|
||||
"prefix": "",
|
||||
"status": "Enabled",
|
||||
"transition": []
|
||||
}
|
||||
],
|
||||
"timeouts": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=",
|
||||
"dependencies": [
|
||||
"aws_s3_bucket.athena_results"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket_notification",
|
||||
"name": "trigger_etl_lambda",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"bucket": "datalake-raw-205216182560-agb-s1201687",
|
||||
"eventbridge": false,
|
||||
"id": "datalake-raw-205216182560-agb-s1201687",
|
||||
"lambda_function": [
|
||||
{
|
||||
"events": [
|
||||
"s3:ObjectCreated:*"
|
||||
],
|
||||
"filter_prefix": "raw-zone/",
|
||||
"filter_suffix": "",
|
||||
"id": "tf-s3-lambda-20240530091945493300000001",
|
||||
"lambda_function_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687"
|
||||
}
|
||||
],
|
||||
"queue": [],
|
||||
"topic": []
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_lambda_function.etl_post_processing",
|
||||
"aws_lambda_layer_version.aws_wrangler",
|
||||
"aws_lambda_permission.allow_bucket",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_ssm_parameter",
|
||||
"name": "s3_processed",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"allowed_pattern": "",
|
||||
"arn": "arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name",
|
||||
"data_type": "text",
|
||||
"description": "",
|
||||
"id": "s3_processed_bucket_name",
|
||||
"insecure_value": null,
|
||||
"key_id": "",
|
||||
"name": "s3_processed_bucket_name",
|
||||
"overwrite": null,
|
||||
"tags": null,
|
||||
"tags_all": {},
|
||||
"tier": "Standard",
|
||||
"type": "String",
|
||||
"value": "datalake-processed-205216182560-agb-s1201687",
|
||||
"version": 1
|
||||
},
|
||||
"sensitive_attributes": [
|
||||
[
|
||||
{
|
||||
"type": "get_attr",
|
||||
"value": "value"
|
||||
}
|
||||
]
|
||||
],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_s3_bucket.processed_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"check_results": null
|
||||
}
|
||||
|
||||
@ -1,896 +1,9 @@
|
||||
{
|
||||
"version": 4,
|
||||
"terraform_version": "1.8.1",
|
||||
"serial": 272,
|
||||
"serial": 323,
|
||||
"lineage": "3ca48c2b-bb88-35e7-8a35-5ef0e08daaff",
|
||||
"outputs": {},
|
||||
"resources": [
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_athena_workgroup",
|
||||
"name": "athena_workgroup",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:athena:us-east-1:205216182560:workgroup/development",
|
||||
"configuration": [
|
||||
{
|
||||
"bytes_scanned_cutoff_per_query": 0,
|
||||
"enforce_workgroup_configuration": true,
|
||||
"engine_version": [
|
||||
{
|
||||
"effective_engine_version": "Athena engine version 3",
|
||||
"selected_engine_version": "AUTO"
|
||||
}
|
||||
],
|
||||
"execution_role": "",
|
||||
"publish_cloudwatch_metrics_enabled": true,
|
||||
"requester_pays_enabled": false,
|
||||
"result_configuration": [
|
||||
{
|
||||
"acl_configuration": [],
|
||||
"encryption_configuration": [],
|
||||
"expected_bucket_owner": "",
|
||||
"output_location": "s3://athena-results-205216182560-agb-s1201687/output/"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"force_destroy": true,
|
||||
"id": "development",
|
||||
"name": "development",
|
||||
"state": "ENABLED",
|
||||
"tags": {},
|
||||
"tags_all": {}
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_s3_bucket.athena_results"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_glue_catalog_database",
|
||||
"name": "datalake_db_processed_zone",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_processed_205216182560_agb_s1201687",
|
||||
"catalog_id": "205216182560",
|
||||
"create_table_default_permission": [
|
||||
{
|
||||
"permissions": [
|
||||
"ALL"
|
||||
],
|
||||
"principal": [
|
||||
{
|
||||
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"federated_database": [],
|
||||
"id": "205216182560:datalake_processed_205216182560_agb_s1201687",
|
||||
"location_uri": "",
|
||||
"name": "datalake_processed_205216182560_agb_s1201687",
|
||||
"parameters": {},
|
||||
"tags": {},
|
||||
"tags_all": {},
|
||||
"target_database": []
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_glue_catalog_database",
|
||||
"name": "datalake_db_raw_zone",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:glue:us-east-1:205216182560:database/datalake_raw_205216182560_agb_s1201687",
|
||||
"catalog_id": "205216182560",
|
||||
"create_table_default_permission": [
|
||||
{
|
||||
"permissions": [
|
||||
"ALL"
|
||||
],
|
||||
"principal": [
|
||||
{
|
||||
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"description": "",
|
||||
"federated_database": [],
|
||||
"id": "205216182560:datalake_raw_205216182560_agb_s1201687",
|
||||
"location_uri": "",
|
||||
"name": "datalake_raw_205216182560_agb_s1201687",
|
||||
"parameters": {},
|
||||
"tags": {},
|
||||
"tags_all": {},
|
||||
"target_database": []
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_glue_crawler",
|
||||
"name": "glue_crawler_raw_zone",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:glue:us-east-1:205216182560:crawler/gc-raw-205216182560-agb-s1201687",
|
||||
"catalog_target": [],
|
||||
"classifiers": [],
|
||||
"configuration": "",
|
||||
"database_name": "datalake_raw_205216182560_agb_s1201687",
|
||||
"delta_target": [],
|
||||
"description": "",
|
||||
"dynamodb_target": [],
|
||||
"hudi_target": [],
|
||||
"iceberg_target": [],
|
||||
"id": "gc-raw-205216182560-agb-s1201687",
|
||||
"jdbc_target": [],
|
||||
"lake_formation_configuration": [
|
||||
{
|
||||
"account_id": "",
|
||||
"use_lake_formation_credentials": false
|
||||
}
|
||||
],
|
||||
"lineage_configuration": [
|
||||
{
|
||||
"crawler_lineage_settings": "DISABLE"
|
||||
}
|
||||
],
|
||||
"mongodb_target": [],
|
||||
"name": "gc-raw-205216182560-agb-s1201687",
|
||||
"recrawl_policy": [
|
||||
{
|
||||
"recrawl_behavior": "CRAWL_EVERYTHING"
|
||||
}
|
||||
],
|
||||
"role": "LabRole",
|
||||
"s3_target": [
|
||||
{
|
||||
"connection_name": "",
|
||||
"dlq_event_queue_arn": "",
|
||||
"event_queue_arn": "",
|
||||
"exclusions": [],
|
||||
"path": "s3://datalake-raw-205216182560-agb-s1201687/raw-zone/stockdata/",
|
||||
"sample_size": 0
|
||||
}
|
||||
],
|
||||
"schedule": "",
|
||||
"schema_change_policy": [
|
||||
{
|
||||
"delete_behavior": "DEPRECATE_IN_DATABASE",
|
||||
"update_behavior": "UPDATE_IN_DATABASE"
|
||||
}
|
||||
],
|
||||
"security_configuration": "",
|
||||
"table_prefix": "crawler_",
|
||||
"tags": {
|
||||
"environment": "DEV",
|
||||
"owner": "Agnieszka Gąbka-Buszek",
|
||||
"purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"environment": "DEV",
|
||||
"owner": "Agnieszka Gąbka-Buszek",
|
||||
"purpose": "UAM Cloud Data Processing"
|
||||
}
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_glue_catalog_database.datalake_db_raw_zone",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_iam_policy",
|
||||
"name": "lambda_policy",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:iam::205216182560:policy/lambda_policy",
|
||||
"attachment_count": 0,
|
||||
"description": "",
|
||||
"id": "arn:aws:iam::205216182560:policy/lambda_policy",
|
||||
"name": "lambda_policy",
|
||||
"name_prefix": "",
|
||||
"path": "/",
|
||||
"policy": "{\"Statement\":[{\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Effect\":\"Allow\",\"Resource\":[\"arn:aws:s3:::datalake-raw-205216182560-agb-s1201687/*\",\"arn:aws:s3:::datalake-processed-205216182560-agb-s1201687/*\"]},{\"Action\":\"ssm:GetParameter\",\"Effect\":\"Allow\",\"Resource\":\"arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name\"}],\"Version\":\"2012-10-17\"}",
|
||||
"policy_id": "ANPAS7R6WOEQGHVSNQS6X",
|
||||
"tags": {},
|
||||
"tags_all": {}
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_s3_bucket.processed_bucket",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_kinesis_firehose_delivery_stream",
|
||||
"name": "stock_delivery_stream",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687",
|
||||
"destination": "extended_s3",
|
||||
"destination_id": "destinationId-000000000001",
|
||||
"elasticsearch_configuration": [],
|
||||
"extended_s3_configuration": [
|
||||
{
|
||||
"bucket_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
|
||||
"buffering_interval": 60,
|
||||
"buffering_size": 1,
|
||||
"cloudwatch_logging_options": [
|
||||
{
|
||||
"enabled": false,
|
||||
"log_group_name": "",
|
||||
"log_stream_name": ""
|
||||
}
|
||||
],
|
||||
"compression_format": "UNCOMPRESSED",
|
||||
"custom_time_zone": "UTC",
|
||||
"data_format_conversion_configuration": [],
|
||||
"dynamic_partitioning_configuration": [],
|
||||
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
||||
"file_extension": "",
|
||||
"kms_key_arn": "",
|
||||
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
|
||||
"processing_configuration": [
|
||||
{
|
||||
"enabled": false,
|
||||
"processors": []
|
||||
}
|
||||
],
|
||||
"role_arn": "arn:aws:iam::205216182560:role/LabRole",
|
||||
"s3_backup_configuration": [],
|
||||
"s3_backup_mode": "Disabled"
|
||||
}
|
||||
],
|
||||
"http_endpoint_configuration": [],
|
||||
"id": "arn:aws:firehose:us-east-1:205216182560:deliverystream/firehose-205216182560-agb-s1201687",
|
||||
"kinesis_source_configuration": [
|
||||
{
|
||||
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
|
||||
"role_arn": "arn:aws:iam::205216182560:role/LabRole"
|
||||
}
|
||||
],
|
||||
"msk_source_configuration": [],
|
||||
"name": "firehose-205216182560-agb-s1201687",
|
||||
"opensearch_configuration": [],
|
||||
"opensearchserverless_configuration": [],
|
||||
"redshift_configuration": [],
|
||||
"server_side_encryption": [
|
||||
{
|
||||
"enabled": false,
|
||||
"key_arn": "",
|
||||
"key_type": "AWS_OWNED_CMK"
|
||||
}
|
||||
],
|
||||
"snowflake_configuration": [],
|
||||
"splunk_configuration": [],
|
||||
"tags": {},
|
||||
"tags_all": {},
|
||||
"timeouts": null,
|
||||
"version_id": "1"
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
|
||||
"dependencies": [
|
||||
"aws_kinesis_stream.cryptostock_stream",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_kinesis_stream",
|
||||
"name": "cryptostock_stream",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 1,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
|
||||
"encryption_type": "NONE",
|
||||
"enforce_consumer_deletion": true,
|
||||
"id": "arn:aws:kinesis:us-east-1:205216182560:stream/cryptostock-205216182560-agb-s1201687",
|
||||
"kms_key_id": "",
|
||||
"name": "cryptostock-205216182560-agb-s1201687",
|
||||
"retention_period": 24,
|
||||
"shard_count": 1,
|
||||
"shard_level_metrics": [
|
||||
"IncomingBytes",
|
||||
"IncomingRecords",
|
||||
"OutgoingBytes",
|
||||
"OutgoingRecords"
|
||||
],
|
||||
"stream_mode_details": [
|
||||
{
|
||||
"stream_mode": "PROVISIONED"
|
||||
}
|
||||
],
|
||||
"tags": {
|
||||
"Environment": "DEV",
|
||||
"Owner": "Agnieszka Gąbka-Buszek",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"Environment": "DEV",
|
||||
"Owner": "Agnieszka Gąbka-Buszek",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"timeouts": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_lambda_function",
|
||||
"name": "etl_post_processing",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"architectures": [
|
||||
"x86_64"
|
||||
],
|
||||
"arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687",
|
||||
"code_signing_config_arn": "",
|
||||
"dead_letter_config": [],
|
||||
"description": "",
|
||||
"environment": [],
|
||||
"ephemeral_storage": [
|
||||
{
|
||||
"size": 512
|
||||
}
|
||||
],
|
||||
"file_system_config": [],
|
||||
"filename": "../lambda/lambda_definition.zip",
|
||||
"function_name": "etl-post-processing-205216182560-agb-s1201687",
|
||||
"handler": "lambda_definition.etl_function",
|
||||
"id": "etl-post-processing-205216182560-agb-s1201687",
|
||||
"image_config": [],
|
||||
"image_uri": "",
|
||||
"invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687/invocations",
|
||||
"kms_key_arn": "",
|
||||
"last_modified": "2024-05-27T10:23:37.000+0000",
|
||||
"layers": [
|
||||
"arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7"
|
||||
],
|
||||
"logging_config": [
|
||||
{
|
||||
"application_log_level": "",
|
||||
"log_format": "Text",
|
||||
"log_group": "/aws/lambda/etl-post-processing-205216182560-agb-s1201687",
|
||||
"system_log_level": ""
|
||||
}
|
||||
],
|
||||
"memory_size": 512,
|
||||
"package_type": "Zip",
|
||||
"publish": false,
|
||||
"qualified_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST",
|
||||
"qualified_invoke_arn": "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687:$LATEST/invocations",
|
||||
"replace_security_groups_on_destroy": null,
|
||||
"replacement_security_group_ids": null,
|
||||
"reserved_concurrent_executions": -1,
|
||||
"role": "arn:aws:iam::205216182560:role/LabRole",
|
||||
"runtime": "python3.8",
|
||||
"s3_bucket": null,
|
||||
"s3_key": null,
|
||||
"s3_object_version": null,
|
||||
"signing_job_arn": "",
|
||||
"signing_profile_version_arn": "",
|
||||
"skip_destroy": false,
|
||||
"snap_start": [],
|
||||
"source_code_hash": "DYklWA51/+hutwYtHutJg59rV7DY0LEgfp+ne8wgiSo=",
|
||||
"source_code_size": 884,
|
||||
"tags": {},
|
||||
"tags_all": {},
|
||||
"timeout": 300,
|
||||
"timeouts": null,
|
||||
"tracing_config": [
|
||||
{
|
||||
"mode": "PassThrough"
|
||||
}
|
||||
],
|
||||
"version": "$LATEST",
|
||||
"vpc_config": []
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6NjAwMDAwMDAwMDAwLCJ1cGRhdGUiOjYwMDAwMDAwMDAwMH19",
|
||||
"dependencies": [
|
||||
"aws_lambda_layer_version.aws_wrangler"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_lambda_layer_version",
|
||||
"name": "aws_wrangler",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7",
|
||||
"compatible_architectures": [],
|
||||
"compatible_runtimes": [
|
||||
"python3.8"
|
||||
],
|
||||
"created_date": "2024-05-27T08:03:57.293+0000",
|
||||
"description": "",
|
||||
"filename": "../lambda/awswrangler-layer-2.7.0-py3.8.zip",
|
||||
"id": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687:7",
|
||||
"layer_arn": "arn:aws:lambda:us-east-1:205216182560:layer:aws_wrangler_205216182560_agb_s1201687",
|
||||
"layer_name": "aws_wrangler_205216182560_agb_s1201687",
|
||||
"license_info": "",
|
||||
"s3_bucket": null,
|
||||
"s3_key": null,
|
||||
"s3_object_version": null,
|
||||
"signing_job_arn": "",
|
||||
"signing_profile_version_arn": "",
|
||||
"skip_destroy": false,
|
||||
"source_code_hash": "C0YX/4auMnBs4J9JCDy1f7uc2GLF0vU7ppQgzffQiN4=",
|
||||
"source_code_size": 43879070,
|
||||
"version": "7"
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA=="
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_lambda_permission",
|
||||
"name": "allow_bucket",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"action": "lambda:InvokeFunction",
|
||||
"event_source_token": null,
|
||||
"function_name": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687",
|
||||
"function_url_auth_type": null,
|
||||
"id": "AllowExecutionFromS3Bucket",
|
||||
"principal": "s3.amazonaws.com",
|
||||
"principal_org_id": null,
|
||||
"qualifier": "",
|
||||
"source_account": null,
|
||||
"source_arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
|
||||
"statement_id": "AllowExecutionFromS3Bucket",
|
||||
"statement_id_prefix": ""
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_lambda_function.etl_post_processing",
|
||||
"aws_lambda_layer_version.aws_wrangler",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket",
|
||||
"name": "athena_results",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"acceleration_status": "",
|
||||
"acl": null,
|
||||
"arn": "arn:aws:s3:::athena-results-205216182560-agb-s1201687",
|
||||
"bucket": "athena-results-205216182560-agb-s1201687",
|
||||
"bucket_domain_name": "athena-results-205216182560-agb-s1201687.s3.amazonaws.com",
|
||||
"bucket_prefix": "",
|
||||
"bucket_regional_domain_name": "athena-results-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
|
||||
"cors_rule": [],
|
||||
"force_destroy": true,
|
||||
"grant": [
|
||||
{
|
||||
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
|
||||
"permissions": [
|
||||
"FULL_CONTROL"
|
||||
],
|
||||
"type": "CanonicalUser",
|
||||
"uri": ""
|
||||
}
|
||||
],
|
||||
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||
"id": "athena-results-205216182560-agb-s1201687",
|
||||
"lifecycle_rule": [
|
||||
{
|
||||
"abort_incomplete_multipart_upload_days": 0,
|
||||
"enabled": true,
|
||||
"expiration": [
|
||||
{
|
||||
"date": "",
|
||||
"days": 1,
|
||||
"expired_object_delete_marker": false
|
||||
}
|
||||
],
|
||||
"id": "standard-expiration",
|
||||
"noncurrent_version_expiration": [],
|
||||
"noncurrent_version_transition": [],
|
||||
"prefix": "",
|
||||
"tags": {},
|
||||
"transition": []
|
||||
}
|
||||
],
|
||||
"logging": [],
|
||||
"object_lock_configuration": [],
|
||||
"object_lock_enabled": false,
|
||||
"policy": "",
|
||||
"region": "us-east-1",
|
||||
"replication_configuration": [],
|
||||
"request_payer": "BucketOwner",
|
||||
"server_side_encryption_configuration": [
|
||||
{
|
||||
"rule": [
|
||||
{
|
||||
"apply_server_side_encryption_by_default": [
|
||||
{
|
||||
"kms_master_key_id": "",
|
||||
"sse_algorithm": "AES256"
|
||||
}
|
||||
],
|
||||
"bucket_key_enabled": false
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": {
|
||||
"environment": "DEV",
|
||||
"owner": "Agnieszka Gąbka-Buszek",
|
||||
"purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"environment": "DEV",
|
||||
"owner": "Agnieszka Gąbka-Buszek",
|
||||
"purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"timeouts": null,
|
||||
"versioning": [
|
||||
{
|
||||
"enabled": false,
|
||||
"mfa_delete": false
|
||||
}
|
||||
],
|
||||
"website": [],
|
||||
"website_domain": null,
|
||||
"website_endpoint": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket",
|
||||
"name": "processed_bucket",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"acceleration_status": "",
|
||||
"acl": null,
|
||||
"arn": "arn:aws:s3:::datalake-processed-205216182560-agb-s1201687",
|
||||
"bucket": "datalake-processed-205216182560-agb-s1201687",
|
||||
"bucket_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.amazonaws.com",
|
||||
"bucket_prefix": "",
|
||||
"bucket_regional_domain_name": "datalake-processed-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
|
||||
"cors_rule": [],
|
||||
"force_destroy": true,
|
||||
"grant": [
|
||||
{
|
||||
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
|
||||
"permissions": [
|
||||
"FULL_CONTROL"
|
||||
],
|
||||
"type": "CanonicalUser",
|
||||
"uri": ""
|
||||
}
|
||||
],
|
||||
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||
"id": "datalake-processed-205216182560-agb-s1201687",
|
||||
"lifecycle_rule": [],
|
||||
"logging": [],
|
||||
"object_lock_configuration": [],
|
||||
"object_lock_enabled": false,
|
||||
"policy": "",
|
||||
"region": "us-east-1",
|
||||
"replication_configuration": [],
|
||||
"request_payer": "BucketOwner",
|
||||
"server_side_encryption_configuration": [
|
||||
{
|
||||
"rule": [
|
||||
{
|
||||
"apply_server_side_encryption_by_default": [
|
||||
{
|
||||
"kms_master_key_id": "",
|
||||
"sse_algorithm": "AES256"
|
||||
}
|
||||
],
|
||||
"bucket_key_enabled": false
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": {
|
||||
"Environment": "DEV",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"Environment": "DEV",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"timeouts": null,
|
||||
"versioning": [
|
||||
{
|
||||
"enabled": false,
|
||||
"mfa_delete": false
|
||||
}
|
||||
],
|
||||
"website": [],
|
||||
"website_domain": null,
|
||||
"website_endpoint": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket",
|
||||
"name": "raw_bucket",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"acceleration_status": "",
|
||||
"acl": null,
|
||||
"arn": "arn:aws:s3:::datalake-raw-205216182560-agb-s1201687",
|
||||
"bucket": "datalake-raw-205216182560-agb-s1201687",
|
||||
"bucket_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.amazonaws.com",
|
||||
"bucket_prefix": "",
|
||||
"bucket_regional_domain_name": "datalake-raw-205216182560-agb-s1201687.s3.us-east-1.amazonaws.com",
|
||||
"cors_rule": [],
|
||||
"force_destroy": true,
|
||||
"grant": [
|
||||
{
|
||||
"id": "42e379e111382262c89fb5a8aef42c2fd8d1e15971b4e7ae2317f0b554d6f32e",
|
||||
"permissions": [
|
||||
"FULL_CONTROL"
|
||||
],
|
||||
"type": "CanonicalUser",
|
||||
"uri": ""
|
||||
}
|
||||
],
|
||||
"hosted_zone_id": "Z3AQBSTGFYJSTF",
|
||||
"id": "datalake-raw-205216182560-agb-s1201687",
|
||||
"lifecycle_rule": [],
|
||||
"logging": [],
|
||||
"object_lock_configuration": [],
|
||||
"object_lock_enabled": false,
|
||||
"policy": "",
|
||||
"region": "us-east-1",
|
||||
"replication_configuration": [],
|
||||
"request_payer": "BucketOwner",
|
||||
"server_side_encryption_configuration": [
|
||||
{
|
||||
"rule": [
|
||||
{
|
||||
"apply_server_side_encryption_by_default": [
|
||||
{
|
||||
"kms_master_key_id": "",
|
||||
"sse_algorithm": "AES256"
|
||||
}
|
||||
],
|
||||
"bucket_key_enabled": false
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"tags": {
|
||||
"Environment": "DEV",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"tags_all": {
|
||||
"Environment": "DEV",
|
||||
"Purpose": "UAM Cloud Data Processing"
|
||||
},
|
||||
"timeouts": null,
|
||||
"versioning": [
|
||||
{
|
||||
"enabled": false,
|
||||
"mfa_delete": false
|
||||
}
|
||||
],
|
||||
"website": [],
|
||||
"website_domain": null,
|
||||
"website_endpoint": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket_lifecycle_configuration",
|
||||
"name": "athena_results_lifecycle",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"bucket": "athena-results-205216182560-agb-s1201687",
|
||||
"expected_bucket_owner": "",
|
||||
"id": "athena-results-205216182560-agb-s1201687",
|
||||
"rule": [
|
||||
{
|
||||
"abort_incomplete_multipart_upload": [],
|
||||
"expiration": [
|
||||
{
|
||||
"date": "",
|
||||
"days": 1,
|
||||
"expired_object_delete_marker": false
|
||||
}
|
||||
],
|
||||
"filter": [
|
||||
{
|
||||
"and": [],
|
||||
"object_size_greater_than": "",
|
||||
"object_size_less_than": "",
|
||||
"prefix": "",
|
||||
"tag": []
|
||||
}
|
||||
],
|
||||
"id": "standard-expiration",
|
||||
"noncurrent_version_expiration": [],
|
||||
"noncurrent_version_transition": [],
|
||||
"prefix": "",
|
||||
"status": "Enabled",
|
||||
"transition": []
|
||||
}
|
||||
],
|
||||
"timeouts": null
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAsInVwZGF0ZSI6MTgwMDAwMDAwMDAwfX0=",
|
||||
"dependencies": [
|
||||
"aws_s3_bucket.athena_results"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_s3_bucket_notification",
|
||||
"name": "trigger_etl_lambda",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"bucket": "datalake-raw-205216182560-agb-s1201687",
|
||||
"eventbridge": false,
|
||||
"id": "datalake-raw-205216182560-agb-s1201687",
|
||||
"lambda_function": [
|
||||
{
|
||||
"events": [
|
||||
"s3:ObjectCreated:*"
|
||||
],
|
||||
"filter_prefix": "raw-zone/",
|
||||
"filter_suffix": "",
|
||||
"id": "tf-s3-lambda-20240527080407879000000001",
|
||||
"lambda_function_arn": "arn:aws:lambda:us-east-1:205216182560:function:etl-post-processing-205216182560-agb-s1201687"
|
||||
}
|
||||
],
|
||||
"queue": [],
|
||||
"topic": []
|
||||
},
|
||||
"sensitive_attributes": [],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_lambda_function.etl_post_processing",
|
||||
"aws_lambda_layer_version.aws_wrangler",
|
||||
"aws_lambda_permission.allow_bucket",
|
||||
"aws_s3_bucket.raw_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"mode": "managed",
|
||||
"type": "aws_ssm_parameter",
|
||||
"name": "s3_processed",
|
||||
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
|
||||
"instances": [
|
||||
{
|
||||
"schema_version": 0,
|
||||
"attributes": {
|
||||
"allowed_pattern": "",
|
||||
"arn": "arn:aws:ssm:us-east-1:205216182560:parameter/s3_processed_bucket_name",
|
||||
"data_type": "text",
|
||||
"description": "",
|
||||
"id": "s3_processed_bucket_name",
|
||||
"insecure_value": null,
|
||||
"key_id": "",
|
||||
"name": "s3_processed_bucket_name",
|
||||
"overwrite": null,
|
||||
"tags": {},
|
||||
"tags_all": {},
|
||||
"tier": "Standard",
|
||||
"type": "String",
|
||||
"value": "datalake-processed-205216182560-agb-s1201687",
|
||||
"version": 1
|
||||
},
|
||||
"sensitive_attributes": [
|
||||
[
|
||||
{
|
||||
"type": "get_attr",
|
||||
"value": "value"
|
||||
}
|
||||
]
|
||||
],
|
||||
"private": "bnVsbA==",
|
||||
"dependencies": [
|
||||
"aws_s3_bucket.processed_bucket"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"resources": [],
|
||||
"check_results": null
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user