diff --git a/labs/data_generator/terraform.tfstate b/labs/data_generator/terraform.tfstate new file mode 100644 index 0000000..496ace8 --- /dev/null +++ b/labs/data_generator/terraform.tfstate @@ -0,0 +1,9 @@ +{ + "version": 4, + "terraform_version": "1.8.1", + "serial": 1, + "lineage": "2510680a-e84f-7a53-abf1-51dd5e72ea27", + "outputs": {}, + "resources": [], + "check_results": null +} diff --git a/labs/terraform/glue_dc.tf b/labs/terraform/glue_dc.tf new file mode 100644 index 0000000..9e12d74 --- /dev/null +++ b/labs/terraform/glue_dc.tf @@ -0,0 +1,3 @@ +resource "aws_glue_catalog_database" "datalake_db_raw_zone" { + name = "datalake_raw_${var.account_number}_${var.student_initials}_${var.student_index_no}" +} \ No newline at end of file diff --git a/labs/terraform/kinesis_ds.tf b/labs/terraform/kinesis_ds.tf new file mode 100644 index 0000000..b93605f --- /dev/null +++ b/labs/terraform/kinesis_ds.tf @@ -0,0 +1,16 @@ +resource "aws_kinesis_stream" "cryptostock_stream" { + name = "cryptostock-${var.account_number}-${var.student_initials}-${var.student_index_no}" + shard_count = 1 + enforce_consumer_deletion = true + shard_level_metrics = [ + "IncomingBytes", + "OutgoingBytes", + "IncomingRecords", + "OutgoingRecords" + ] + tags = { + Purpose = "UAM Cloud Data Processing" + Environment = "DEV" + Owner = var.student_full_name + } +} \ No newline at end of file diff --git a/labs/terraform/kinesis_fh.tf b/labs/terraform/kinesis_fh.tf new file mode 100644 index 0000000..73e9aa1 --- /dev/null +++ b/labs/terraform/kinesis_fh.tf @@ -0,0 +1,16 @@ +resource "aws_kinesis_firehose_delivery_stream" "stock_delivery_stream" { + name = "firehose-${var.account_number}-${var.student_initials}-${var.student_index_no}" + destination = "extended_s3" + kinesis_source_configuration { + kinesis_stream_arn = aws_kinesis_stream.cryptostock_stream.arn + role_arn = var.lab_role_arn + } + extended_s3_configuration { + role_arn = var.lab_role_arn + bucket_arn = aws_s3_bucket.raw_bucket.arn + buffering_size = 1 + buffering_interval = 60 + prefix = "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/" + error_output_prefix ="${"raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}"}${"/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}"}/" + } +} \ No newline at end of file diff --git a/labs/terraform/s3.tf b/labs/terraform/s3.tf new file mode 100644 index 0000000..b0ebade --- /dev/null +++ b/labs/terraform/s3.tf @@ -0,0 +1,20 @@ +resource "aws_s3_bucket" "raw_bucket" { + bucket = "datalake-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}" + force_destroy = true + + tags = { + Purpose = "UAM Cloud Data Processing" + Environment = "DEV" + } +} + + +resource "aws_s3_bucket" "processed_bucket" { + bucket = "datalake-processed-${var.account_number}-${var.student_initials}-${var.student_index_no}" + force_destroy = true + + tags = { + Purpose = "UAM Cloud Data Processing" + Environment = "DEV" + } +} \ No newline at end of file diff --git a/labs/terraform/terraform.tfstate b/labs/terraform/terraform.tfstate new file mode 100644 index 0000000..4274840 --- /dev/null +++ b/labs/terraform/terraform.tfstate @@ -0,0 +1,9 @@ +{ + "version": 4, + "terraform_version": "1.8.1", + "serial": 25, + "lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2", + "outputs": {}, + "resources": [], + "check_results": null +} diff --git a/labs/terraform/terraform.tfstate.backup b/labs/terraform/terraform.tfstate.backup new file mode 100644 index 0000000..9049295 --- /dev/null +++ b/labs/terraform/terraform.tfstate.backup @@ -0,0 +1,327 @@ +{ + "version": 4, + "terraform_version": "1.8.1", + "serial": 19, + "lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "aws_glue_catalog_database", + "name": "datalake_db_raw_zone", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680", + "catalog_id": "534534002841", + "create_table_default_permission": [ + { + "permissions": [ + "ALL" + ], + "principal": [ + { + "data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS" + } + ] + } + ], + "description": "", + "federated_database": [], + "id": "534534002841:datalake_raw_534534002841_ab_1201680", + "location_uri": "", + "name": "datalake_raw_534534002841_ab_1201680", + "parameters": null, + "tags": null, + "tags_all": {}, + "target_database": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==" + } + ] + }, + { + "mode": "managed", + "type": "aws_kinesis_firehose_delivery_stream", + "name": "stock_delivery_stream", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", + "destination": "extended_s3", + "destination_id": "destinationId-000000000001", + "elasticsearch_configuration": [], + "extended_s3_configuration": [ + { + "bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", + "buffering_interval": 60, + "buffering_size": 1, + "cloudwatch_logging_options": [ + { + "enabled": false, + "log_group_name": "", + "log_stream_name": "" + } + ], + "compression_format": "UNCOMPRESSED", + "custom_time_zone": "UTC", + "data_format_conversion_configuration": [], + "dynamic_partitioning_configuration": [], + "error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", + "file_extension": "", + "kms_key_arn": "", + "prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/", + "processing_configuration": [ + { + "enabled": false, + "processors": [] + } + ], + "role_arn": "arn:aws:iam::534534002841:role/LabRole", + "s3_backup_configuration": [], + "s3_backup_mode": "Disabled" + } + ], + "http_endpoint_configuration": [], + "id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680", + "kinesis_source_configuration": [ + { + "kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "role_arn": "arn:aws:iam::534534002841:role/LabRole" + } + ], + "msk_source_configuration": [], + "name": "firehose-534534002841-ab-1201680", + "opensearch_configuration": [], + "opensearchserverless_configuration": [], + "redshift_configuration": [], + "server_side_encryption": [ + { + "enabled": false, + "key_arn": "", + "key_type": "AWS_OWNED_CMK" + } + ], + "snowflake_configuration": [], + "splunk_configuration": [], + "tags": {}, + "tags_all": {}, + "timeouts": null, + "version_id": "1" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=", + "dependencies": [ + "aws_kinesis_stream.cryptostock_stream", + "aws_s3_bucket.raw_bucket" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_kinesis_stream", + "name": "cryptostock_stream", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "encryption_type": "NONE", + "enforce_consumer_deletion": true, + "id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680", + "kms_key_id": "", + "name": "cryptostock-534534002841-ab-1201680", + "retention_period": 24, + "shard_count": 1, + "shard_level_metrics": [ + "IncomingBytes", + "IncomingRecords", + "OutgoingBytes", + "OutgoingRecords" + ], + "stream_mode_details": [ + { + "stream_mode": "PROVISIONED" + } + ], + "tags": { + "Environment": "DEV", + "Owner": "Aron Boguszewski", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Owner": "Aron Boguszewski", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "processed_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680", + "bucket": "datalake-processed-534534002841-ab-1201680", + "bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "datalake-processed-534534002841-ab-1201680", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + }, + { + "mode": "managed", + "type": "aws_s3_bucket", + "name": "raw_bucket", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "acceleration_status": "", + "acl": null, + "arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680", + "bucket": "datalake-raw-534534002841-ab-1201680", + "bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com", + "bucket_prefix": "", + "bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com", + "cors_rule": [], + "force_destroy": true, + "grant": [ + { + "id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1", + "permissions": [ + "FULL_CONTROL" + ], + "type": "CanonicalUser", + "uri": "" + } + ], + "hosted_zone_id": "Z3AQBSTGFYJSTF", + "id": "datalake-raw-534534002841-ab-1201680", + "lifecycle_rule": [], + "logging": [], + "object_lock_configuration": [], + "object_lock_enabled": false, + "policy": "", + "region": "us-east-1", + "replication_configuration": [], + "request_payer": "BucketOwner", + "server_side_encryption_configuration": [ + { + "rule": [ + { + "apply_server_side_encryption_by_default": [ + { + "kms_master_key_id": "", + "sse_algorithm": "AES256" + } + ], + "bucket_key_enabled": false + } + ] + } + ], + "tags": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "tags_all": { + "Environment": "DEV", + "Purpose": "UAM Cloud Data Processing" + }, + "timeouts": null, + "versioning": [ + { + "enabled": false, + "mfa_delete": false + } + ], + "website": [], + "website_domain": null, + "website_endpoint": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/labs/terraform/terraform.tfvars b/labs/terraform/terraform.tfvars index 1950101..3239d71 100644 --- a/labs/terraform/terraform.tfvars +++ b/labs/terraform/terraform.tfvars @@ -1,5 +1,5 @@ -account_number=XXXXX -student_initials="jk" -student_full_name="Jakub Kasprzak" -student_index_no = "12345" -lab_role_arn = "arn:aws:iam::XXXXX:role/LabRole" \ No newline at end of file +account_number=534534002841 +student_initials="ab" +student_full_name="Aron Boguszewski" +student_index_no = "1201680" +lab_role_arn = "arn:aws:iam::534534002841:role/LabRole" \ No newline at end of file