Materiały na zajęcia
This commit is contained in:
parent
7d53b118d7
commit
08662c4f24
@ -1,17 +0,0 @@
|
|||||||
resource "aws_glue_catalog_database" "datalake_db_raw_zone" {
|
|
||||||
name = "datalake_${var.environment}_${var.account_number}_${var.student_initials}_${var.student_index_no}"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
resource "aws_glue_crawler" "glue_crawler_raw_zone" {
|
|
||||||
database_name = aws_glue_catalog_database.datalake_db_raw_zone.name
|
|
||||||
name = "gc-raw-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
role = aws_iam_role.glue_crawler_role.arn
|
|
||||||
table_prefix = "crawler_"
|
|
||||||
|
|
||||||
s3_target {
|
|
||||||
path = "s3://${aws_s3_bucket.main_dl_bucket.bucket}/raw-zone/stockdata/"
|
|
||||||
}
|
|
||||||
|
|
||||||
tags = merge(local.common_tags, )
|
|
||||||
}
|
|
||||||
@ -1,171 +0,0 @@
|
|||||||
resource "aws_iam_role" "firehose_stream_role" {
|
|
||||||
name = "firehose-role-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
|
|
||||||
assume_role_policy = <<EOF
|
|
||||||
{
|
|
||||||
"Version": "2012-10-17",
|
|
||||||
"Statement": [
|
|
||||||
{
|
|
||||||
"Action": "sts:AssumeRole",
|
|
||||||
"Principal": {
|
|
||||||
"Service": "firehose.amazonaws.com"
|
|
||||||
},
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Sid": ""
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "aws_iam_role_policy" "firehose_stream_policy" {
|
|
||||||
name = "firehose-stream-policy-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
role = aws_iam_role.firehose_stream_role.id
|
|
||||||
|
|
||||||
policy = <<EOF
|
|
||||||
{
|
|
||||||
"Version": "2012-10-17",
|
|
||||||
"Statement": [
|
|
||||||
{
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Action": "kinesis:*",
|
|
||||||
"Resource": "*"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Action": [
|
|
||||||
"s3:AbortMultipartUpload",
|
|
||||||
"s3:GetBucketLocation",
|
|
||||||
"s3:GetObject",
|
|
||||||
"s3:ListBucket",
|
|
||||||
"s3:ListBucketMultipartUploads",
|
|
||||||
"s3:PutObject"
|
|
||||||
],
|
|
||||||
"Resource": [
|
|
||||||
"${aws_s3_bucket.main_dl_bucket.arn}",
|
|
||||||
"${aws_s3_bucket.main_dl_bucket.arn}/*"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"Sid": "",
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Action": [
|
|
||||||
"logs:PutLogEvents"
|
|
||||||
],
|
|
||||||
"Resource": [
|
|
||||||
"arn:aws:logs:${var.region}:${var.account_number}:log-group:/aws/kinesisfirehose/*"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// Role & policies for Glue Crawler
|
|
||||||
resource "aws_iam_role" "glue_crawler_role" {
|
|
||||||
name = "crawler-role-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
|
|
||||||
assume_role_policy = <<EOF
|
|
||||||
{
|
|
||||||
"Version": "2012-10-17",
|
|
||||||
"Statement": [
|
|
||||||
{
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Principal": {
|
|
||||||
"Service": "glue.amazonaws.com"
|
|
||||||
},
|
|
||||||
"Action": "sts:AssumeRole"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
data "aws_iam_policy" "glue_service_policy" {
|
|
||||||
arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "aws_iam_role_policy" "glue_crawler_user_bucket_policy" {
|
|
||||||
name = "user-bucket-policy-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
role = aws_iam_role.glue_crawler_role.id
|
|
||||||
|
|
||||||
policy = <<EOF
|
|
||||||
{
|
|
||||||
"Version": "2012-10-17",
|
|
||||||
"Statement": [
|
|
||||||
{
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Action": [
|
|
||||||
"s3:GetObject",
|
|
||||||
"s3:PutObject"
|
|
||||||
],
|
|
||||||
"Resource": [
|
|
||||||
"${aws_s3_bucket.main_dl_bucket.arn}*"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "aws_iam_policy_attachment" "crawler_attach_managed_policy" {
|
|
||||||
name = "crawler-managed-service-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
roles = [
|
|
||||||
aws_iam_role.glue_crawler_role.name]
|
|
||||||
policy_arn = data.aws_iam_policy.glue_service_policy.arn
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// Role and policies for Lambda
|
|
||||||
resource "aws_iam_role" "lambda_basic_role" {
|
|
||||||
name = "lambda-basic-role-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
tags = merge(local.common_tags, )
|
|
||||||
|
|
||||||
assume_role_policy = <<EOF
|
|
||||||
{
|
|
||||||
"Version": "2012-10-17",
|
|
||||||
"Statement": [
|
|
||||||
{
|
|
||||||
"Action": "sts:AssumeRole",
|
|
||||||
"Principal": {
|
|
||||||
"Service": "lambda.amazonaws.com"
|
|
||||||
},
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Sid": ""
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
resource "aws_iam_role_policy" "lambda_basic_policy" {
|
|
||||||
name = "lambda-basic-policy-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
role = aws_iam_role.lambda_basic_role.id
|
|
||||||
|
|
||||||
policy = <<EOF
|
|
||||||
{
|
|
||||||
"Version": "2012-10-17",
|
|
||||||
"Statement": [
|
|
||||||
{
|
|
||||||
"Action": [
|
|
||||||
"logs:CreateLogGroup",
|
|
||||||
"logs:CreateLogStream",
|
|
||||||
"logs:PutLogEvents"
|
|
||||||
],
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Resource": "*"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Action": "s3:*",
|
|
||||||
"Resource": [
|
|
||||||
"${aws_s3_bucket.main_dl_bucket.arn}",
|
|
||||||
"${aws_s3_bucket.main_dl_bucket.arn}/*"]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
EOF
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
resource "aws_kinesis_stream" "cryptostock_stream" {
|
|
||||||
name = "cryptostock-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
shard_count = 1
|
|
||||||
enforce_consumer_deletion = true
|
|
||||||
|
|
||||||
shard_level_metrics = [
|
|
||||||
"IncomingBytes",
|
|
||||||
"OutgoingBytes",
|
|
||||||
"IncomingRecords",
|
|
||||||
"OutgoingRecords"
|
|
||||||
]
|
|
||||||
|
|
||||||
tags = merge(local.common_tags, )
|
|
||||||
}
|
|
||||||
@ -1,18 +0,0 @@
|
|||||||
resource "aws_kinesis_firehose_delivery_stream" "stock_delivery_stream" {
|
|
||||||
name = "firehose-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
destination = "extended_s3"
|
|
||||||
|
|
||||||
kinesis_source_configuration {
|
|
||||||
kinesis_stream_arn = aws_kinesis_stream.cryptostock_stream.arn
|
|
||||||
role_arn = aws_iam_role.firehose_stream_role.arn
|
|
||||||
}
|
|
||||||
|
|
||||||
extended_s3_configuration {
|
|
||||||
role_arn = aws_iam_role.firehose_stream_role.arn
|
|
||||||
bucket_arn = aws_s3_bucket.main_dl_bucket.arn
|
|
||||||
buffer_size = 1
|
|
||||||
buffer_interval = 60
|
|
||||||
prefix = "raw-zone/stockdata/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
|
|
||||||
error_output_prefix = "${ "raw-zone/stockdata_errors/!{firehose:error-output-type}/year=!{timestamp:yyyy}"}${ "/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}"}/"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,50 +0,0 @@
|
|||||||
resource "aws_lambda_layer_version" "aws_wrangler" {
|
|
||||||
filename = "../lambda/awswrangler-layer-2.7.0-py3.8.zip"
|
|
||||||
layer_name = "aws_wrangler_${var.environment}_${var.account_number}_${var.student_initials}_${var.student_index_no}"
|
|
||||||
source_code_hash = "${filebase64sha256("../lambda/awswrangler-layer-2.7.0-py3.8.zip")}"
|
|
||||||
compatible_runtimes = [
|
|
||||||
"python3.8"]
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
resource "aws_lambda_function" "etl_post_processing" {
|
|
||||||
|
|
||||||
function_name = "etl-post-processing-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
filename = "../lambda/lambda_definition.zip"
|
|
||||||
handler = "lambda_definition.etl_function"
|
|
||||||
runtime = "python3.8"
|
|
||||||
role = aws_iam_role.lambda_basic_role.arn
|
|
||||||
timeout = 300
|
|
||||||
memory_size = 512
|
|
||||||
source_code_hash = filebase64sha256("../lambda/lambda_definition.zip")
|
|
||||||
layers = [
|
|
||||||
"${aws_lambda_layer_version.aws_wrangler.arn}"]
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
resource "aws_lambda_permission" "allow_bucket" {
|
|
||||||
statement_id = "AllowExecutionFromS3Bucket"
|
|
||||||
action = "lambda:InvokeFunction"
|
|
||||||
function_name = aws_lambda_function.etl_post_processing.arn
|
|
||||||
principal = "s3.amazonaws.com"
|
|
||||||
source_arn = aws_s3_bucket.main_dl_bucket.arn
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
resource "aws_s3_bucket_notification" "trigger_etl_lambda" {
|
|
||||||
bucket = aws_s3_bucket.main_dl_bucket.id
|
|
||||||
|
|
||||||
lambda_function {
|
|
||||||
lambda_function_arn = aws_lambda_function.etl_post_processing.arn
|
|
||||||
events = [
|
|
||||||
"s3:ObjectCreated:*"]
|
|
||||||
filter_prefix = "raw-zone/"
|
|
||||||
}
|
|
||||||
|
|
||||||
depends_on = [
|
|
||||||
aws_lambda_permission.allow_bucket]
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@ -1,7 +0,0 @@
|
|||||||
locals {
|
|
||||||
common_tags = {
|
|
||||||
Purpose = "UAM Cloud Data Processing"
|
|
||||||
Environment = "DEV"
|
|
||||||
Owner = var.student_full_name
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,75 +0,0 @@
|
|||||||
-- lab 3.3
|
|
||||||
|
|
||||||
WITH CTE AS
|
|
||||||
(
|
|
||||||
|
|
||||||
SELECT date_format(from_unixtime(transaction_ts),'%Y-%m-%dT%H') as HourlyBucket,
|
|
||||||
RANK() OVER(PARTITION BY date_format(from_unixtime(transaction_ts),'%Y-%m-%dT%H'), symbol ,type ORDER BY dollar_amount DESC) as rnk, *
|
|
||||||
FROM "datalake_dev_100603781557_jk_12345"."crawler_stockdata"
|
|
||||||
|
|
||||||
)
|
|
||||||
select *
|
|
||||||
from CTE
|
|
||||||
where rnk=1
|
|
||||||
order by 1, 4, 8
|
|
||||||
|
|
||||||
-- LAB 4.2
|
|
||||||
|
|
||||||
CREATE EXTERNAL TABLE processed_stockdata(
|
|
||||||
transaction_date timestamp,
|
|
||||||
price double,
|
|
||||||
amount double,
|
|
||||||
dollar_amount double,
|
|
||||||
type string,
|
|
||||||
trans_id bigint)
|
|
||||||
PARTITIONED BY (
|
|
||||||
symbol string,
|
|
||||||
year integer,
|
|
||||||
month integer,
|
|
||||||
day integer,
|
|
||||||
hour integer
|
|
||||||
)
|
|
||||||
ROW FORMAT SERDE
|
|
||||||
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
|
|
||||||
STORED AS INPUTFORMAT
|
|
||||||
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
|
|
||||||
OUTPUTFORMAT
|
|
||||||
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
|
|
||||||
LOCATION
|
|
||||||
's3://datalake-dev-100603781557-jk-12345/processed-zone/stockdata/'
|
|
||||||
|
|
||||||
|
|
||||||
MSCK REPAIR TABLE processed_stockdata;
|
|
||||||
|
|
||||||
-- LAB 5
|
|
||||||
|
|
||||||
-- .----------. .----------. .----------.
|
|
||||||
-- | SOURCE | | INSERT | | DESTIN. |
|
|
||||||
-- Source-->| STREAM |-->| & SELECT |-->| STREAM |-->Destination
|
|
||||||
-- | | | (PUMP) | | |
|
|
||||||
-- '----------' '----------' '----------'
|
|
||||||
|
|
||||||
|
|
||||||
CREATE OR REPLACE STREAM "DESTINATION_SQL_STREAM"
|
|
||||||
("symbol" VARCHAR(10), "type" VARCHAR(10), "trans_id" BIGINT,
|
|
||||||
"dollar_amount" DOUBLE, "AvgLast30seconds" DOUBLE, "CntLast30seconds" INT,
|
|
||||||
"SumLast30rows" DOUBLE, "CntLast30rows" INT, "max_tran_id" BIGINT );
|
|
||||||
|
|
||||||
CREATE OR REPLACE PUMP "STREAM_PUMP" AS INSERT INTO "DESTINATION_SQL_STREAM"
|
|
||||||
SELECT STREAM "symbol", "type", "trans_id", "dollar_amount", "AvgLast30seconds", "CntLast30seconds"
|
|
||||||
, "SumLast30rows", "CntLast30rows", "max_tran_id"
|
|
||||||
FROM (
|
|
||||||
|
|
||||||
SELECT STREAM "symbol", "type", "trans_id", "dollar_amount",
|
|
||||||
AVG("dollar_amount") OVER LAST_30_SECS AS "AvgLast30seconds",
|
|
||||||
COUNT(*) OVER LAST_30_SECS AS "CntLast30seconds",
|
|
||||||
SUM("dollar_amount") OVER LAST_30_ROWS AS "SumLast30rows",
|
|
||||||
COUNT(*) OVER LAST_30_ROWS AS "CntLast30rows",
|
|
||||||
MAX("trans_id") OVER LAST_30_ROWS AS "max_tran_id"
|
|
||||||
FROM "SOURCE_SQL_STREAM_001"
|
|
||||||
WHERE "symbol" = 'BTC_USD'
|
|
||||||
WINDOW
|
|
||||||
LAST_30_SECS AS (PARTITION BY "symbol", "type" RANGE INTERVAL '30' SECOND PRECEDING),
|
|
||||||
LAST_30_ROWS AS (PARTITION BY "symbol", "type" ROWS 30 PRECEDING)
|
|
||||||
)
|
|
||||||
WHERE "dollar_amount" > 4 * ("AvgLast30seconds");
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
resource "aws_s3_bucket" "main_dl_bucket" {
|
|
||||||
bucket = "datalake-${var.environment}-${var.account_number}-${var.student_initials}-${var.student_index_no}"
|
|
||||||
force_destroy = true
|
|
||||||
|
|
||||||
tags = merge(local.common_tags, )
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue
Block a user