71 lines
1.5 KiB
Terraform
71 lines
1.5 KiB
Terraform
|
provider "google" {
|
||
|
|
project = var.project_id
|
||
|
|
region = var.region
|
||
|
|
}
|
||
|
|
|
||
|
|
data "google_project" "project" {}
|
||
|
|
|
||
|
|
resource "null_resource" "enable_api" {
|
||
|
|
provisioner "local-exec" {
|
||
|
|
command = "gcloud services enable dataproc.googleapis.com"
|
||
|
|
}
|
||
|
|
depends_on = [
|
||
|
|
google_project_iam_member.service-agent-role
|
||
|
|
]
|
||
|
|
}
|
||
|
|
|
||
|
|
resource "google_dataproc_cluster" "sandbox" {
|
||
|
|
name = var.cluster_name
|
||
|
|
project = var.project_id
|
||
|
|
region = var.region
|
||
|
|
|
||
|
|
cluster_config {
|
||
|
|
|
||
|
|
master_config {
|
||
|
|
num_instances = 1
|
||
|
|
machine_type = var.master_machine_type
|
||
|
|
disk_config {
|
||
|
|
boot_disk_size_gb = 30
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
worker_config {
|
||
|
|
num_instances = 0
|
||
|
|
}
|
||
|
|
|
||
|
|
software_config {
|
||
|
|
override_properties = {
|
||
|
|
"dataproc:dataproc.allow.zero.workers" = "true"
|
||
|
|
}
|
||
|
|
|
||
|
|
optional_components = [
|
||
|
|
"ZEPPELIN",
|
||
|
|
# "JUPYTER",
|
||
|
|
"HIVE_WEBHCAT",
|
||
|
|
# "HUDI"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
|
||
|
|
endpoint_config {
|
||
|
|
enable_http_port_access = "true"
|
||
|
|
}
|
||
|
|
|
||
|
|
lifecycle_config {
|
||
|
|
idle_delete_ttl = var.idle_delete_ttl
|
||
|
|
auto_delete_time = timeadd(timestamp(), var.delete_ttl)
|
||
|
|
}
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
depends_on = [
|
||
|
|
null_resource.enable_api,
|
||
|
|
google_project_iam_member.service-agent-role
|
||
|
|
]
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
resource "google_project_iam_member" "service-agent-role" {
|
||
|
|
member = "serviceAccount:service-${data.google_project.project.number}@dataproc-accounts.iam.gserviceaccount.com"
|
||
|
|
project = data.google_project.project.project_id
|
||
|
|
role = "roles/dataproc.serviceAgent"
|
||
|
|
}
|