s1201680/chmury_aronb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test

Browse Source
This commit is contained in:
EC2 Default User 2024-05-08 18:21:42 +00:00
parent 4da2340ec3
commit 8228ac3005
8 changed files with 405 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
labs/data_generator/terraform.tfstate Normal file
View File

@ -0,0 +1,9 @@
{
"version": 4,
"terraform_version": "1.8.1",
"serial": 1,
"lineage": "2510680a-e84f-7a53-abf1-51dd5e72ea27",
"outputs": {},
"resources": [],
"check_results": null
}

3
labs/terraform/glue_dc.tf Normal file
View File

@ -0,0 +1,3 @@
resource "aws_glue_catalog_database" "datalake_db_raw_zone" {
name = "datalake_raw_${var.account_number}_${var.student_initials}_${var.student_index_no}"
}

16
labs/terraform/kinesis_ds.tf Normal file
View File

@ -0,0 +1,16 @@
resource "aws_kinesis_stream" "cryptostock_stream" {
name = "cryptostock-${var.account_number}-${var.student_initials}-${var.student_index_no}"
shard_count = 1
enforce_consumer_deletion = true
shard_level_metrics = [
"IncomingBytes",
"OutgoingBytes",
"IncomingRecords",
"OutgoingRecords"
]
tags = {
Purpose = "UAM Cloud Data Processing"
Environment = "DEV"
Owner = var.student_full_name
}
}

16
labs/terraform/kinesis_fh.tf Normal file
View File

@ -0,0 +1,16 @@
resource "aws_kinesis_firehose_delivery_stream" "stock_delivery_stream" {
name = "firehose-${var.account_number}-${var.student_initials}-${var.student_index_no}"
destination = "extended_s3"
kinesis_source_configuration {
kinesis_stream_arn = aws_kinesis_stream.cryptostock_stream.arn
role_arn = var.lab_role_arn
}
extended_s3_configuration {
role_arn = var.lab_role_arn
bucket_arn = aws_s3_bucket.raw_bucket.arn
buffering_size = 1
buffering_interval = 60
prefix = "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
error_output_prefix ="${"raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}"}${"/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}"}/"
}
}

20
labs/terraform/s3.tf Normal file
View File

@ -0,0 +1,20 @@
resource "aws_s3_bucket" "raw_bucket" {
bucket = "datalake-raw-${var.account_number}-${var.student_initials}-${var.student_index_no}"
force_destroy = true
tags = {
Purpose = "UAM Cloud Data Processing"
Environment = "DEV"
}
}
resource "aws_s3_bucket" "processed_bucket" {
bucket = "datalake-processed-${var.account_number}-${var.student_initials}-${var.student_index_no}"
force_destroy = true
tags = {
Purpose = "UAM Cloud Data Processing"
Environment = "DEV"
}
}

9
labs/terraform/terraform.tfstate Normal file
View File

@ -0,0 +1,9 @@
{
"version": 4,
"terraform_version": "1.8.1",
"serial": 25,
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
"outputs": {},
"resources": [],
"check_results": null
}

327
labs/terraform/terraform.tfstate.backup Normal file
View File

@ -0,0 +1,327 @@
{
"version": 4,
"terraform_version": "1.8.1",
"serial": 19,
"lineage": "a77aaaba-b4f8-6adb-0387-8f0b98d722c2",
"outputs": {},
"resources": [
{
"mode": "managed",
"type": "aws_glue_catalog_database",
"name": "datalake_db_raw_zone",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"arn": "arn:aws:glue:us-east-1:534534002841:database/datalake_raw_534534002841_ab_1201680",
"catalog_id": "534534002841",
"create_table_default_permission": [
{
"permissions": [
"ALL"
],
"principal": [
{
"data_lake_principal_identifier": "IAM_ALLOWED_PRINCIPALS"
}
]
}
],
"description": "",
"federated_database": [],
"id": "534534002841:datalake_raw_534534002841_ab_1201680",
"location_uri": "",
"name": "datalake_raw_534534002841_ab_1201680",
"parameters": null,
"tags": null,
"tags_all": {},
"target_database": []
},
"sensitive_attributes": [],
"private": "bnVsbA=="
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_firehose_delivery_stream",
"name": "stock_delivery_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"destination": "extended_s3",
"destination_id": "destinationId-000000000001",
"elasticsearch_configuration": [],
"extended_s3_configuration": [
{
"bucket_arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"buffering_interval": 60,
"buffering_size": 1,
"cloudwatch_logging_options": [
{
"enabled": false,
"log_group_name": "",
"log_stream_name": ""
}
],
"compression_format": "UNCOMPRESSED",
"custom_time_zone": "UTC",
"data_format_conversion_configuration": [],
"dynamic_partitioning_configuration": [],
"error_output_prefix": "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"file_extension": "",
"kms_key_arn": "",
"prefix": "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/",
"processing_configuration": [
{
"enabled": false,
"processors": []
}
],
"role_arn": "arn:aws:iam::534534002841:role/LabRole",
"s3_backup_configuration": [],
"s3_backup_mode": "Disabled"
}
],
"http_endpoint_configuration": [],
"id": "arn:aws:firehose:us-east-1:534534002841:deliverystream/firehose-534534002841-ab-1201680",
"kinesis_source_configuration": [
{
"kinesis_stream_arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"role_arn": "arn:aws:iam::534534002841:role/LabRole"
}
],
"msk_source_configuration": [],
"name": "firehose-534534002841-ab-1201680",
"opensearch_configuration": [],
"opensearchserverless_configuration": [],
"redshift_configuration": [],
"server_side_encryption": [
{
"enabled": false,
"key_arn": "",
"key_type": "AWS_OWNED_CMK"
}
],
"snowflake_configuration": [],
"splunk_configuration": [],
"tags": {},
"tags_all": {},
"timeouts": null,
"version_id": "1"
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJkZWxldGUiOjE4MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=",
"dependencies": [
"aws_kinesis_stream.cryptostock_stream",
"aws_s3_bucket.raw_bucket"
]
}
]
},
{
"mode": "managed",
"type": "aws_kinesis_stream",
"name": "cryptostock_stream",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 1,
"attributes": {
"arn": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"encryption_type": "NONE",
"enforce_consumer_deletion": true,
"id": "arn:aws:kinesis:us-east-1:534534002841:stream/cryptostock-534534002841-ab-1201680",
"kms_key_id": "",
"name": "cryptostock-534534002841-ab-1201680",
"retention_period": 24,
"shard_count": 1,
"shard_level_metrics": [
"IncomingBytes",
"IncomingRecords",
"OutgoingBytes",
"OutgoingRecords"
],
"stream_mode_details": [
{
"stream_mode": "PROVISIONED"
}
],
"tags": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Owner": "Aron Boguszewski",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjozMDAwMDAwMDAwMDAsImRlbGV0ZSI6NzIwMDAwMDAwMDAwMCwidXBkYXRlIjo3MjAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0="
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "processed_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-processed-534534002841-ab-1201680",
"bucket": "datalake-processed-534534002841-ab-1201680",
"bucket_domain_name": "datalake-processed-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-processed-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-processed-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
},
{
"mode": "managed",
"type": "aws_s3_bucket",
"name": "raw_bucket",
"provider": "provider[\"registry.terraform.io/hashicorp/aws\"]",
"instances": [
{
"schema_version": 0,
"attributes": {
"acceleration_status": "",
"acl": null,
"arn": "arn:aws:s3:::datalake-raw-534534002841-ab-1201680",
"bucket": "datalake-raw-534534002841-ab-1201680",
"bucket_domain_name": "datalake-raw-534534002841-ab-1201680.s3.amazonaws.com",
"bucket_prefix": "",
"bucket_regional_domain_name": "datalake-raw-534534002841-ab-1201680.s3.us-east-1.amazonaws.com",
"cors_rule": [],
"force_destroy": true,
"grant": [
{
"id": "32cafe8e58f64c6af4c3b5901764d8e8cf28ac459a9dc871cea90627ca5a57e1",
"permissions": [
"FULL_CONTROL"
],
"type": "CanonicalUser",
"uri": ""
}
],
"hosted_zone_id": "Z3AQBSTGFYJSTF",
"id": "datalake-raw-534534002841-ab-1201680",
"lifecycle_rule": [],
"logging": [],
"object_lock_configuration": [],
"object_lock_enabled": false,
"policy": "",
"region": "us-east-1",
"replication_configuration": [],
"request_payer": "BucketOwner",
"server_side_encryption_configuration": [
{
"rule": [
{
"apply_server_side_encryption_by_default": [
{
"kms_master_key_id": "",
"sse_algorithm": "AES256"
}
],
"bucket_key_enabled": false
}
]
}
],
"tags": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"tags_all": {
"Environment": "DEV",
"Purpose": "UAM Cloud Data Processing"
},
"timeouts": null,
"versioning": [
{
"enabled": false,
"mfa_delete": false
}
],
"website": [],
"website_domain": null,
"website_endpoint": null
},
"sensitive_attributes": [],
"private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjM2MDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19"
}
]
}
],
"check_results": null
}

10
labs/terraform/terraform.tfvars
View File

@ -1,5 +1,5 @@
account_number=XXXXX
student_initials="jk"
student_full_name="Jakub Kasprzak"
student_index_no = "12345"
lab_role_arn = "arn:aws:iam::XXXXX:role/LabRole"
account_number=534534002841
student_initials="ab"
student_full_name="Aron Boguszewski"
student_index_no = "1201680"
lab_role_arn = "arn:aws:iam::534534002841:role/LabRole"