CCM-13905: Gateway Improvements

infrastructure/terraform/components/reporting/README.md

@@ -41,11 +41,12 @@ No requirements.
 | <a name="input_periodic_s3backup_copy_retention_days"></a> [periodic\_s3backup\_copy\_retention\_days](#input\_periodic\_s3backup\_copy\_retention\_days) | number of days to retain weekly s3 backups in the destination vault | `number` | `31` | no |
 | <a name="input_periodic_s3backup_retention_days"></a> [periodic\_s3backup\_retention\_days](#input\_periodic\_s3backup\_retention\_days) | number of days to retain weekly s3 backups | `number` | `31` | no |
 | <a name="input_periodic_s3backup_schedule"></a> [periodic\_s3backup\_schedule](#input\_periodic\_s3backup\_schedule) | Crontab formatted schedule for Periodic S3 Backups | `string` | `"cron(0 5 ? * 7 *)"` | no |
+| <a name="input_powerbi_gateway_instance_count"></a> [powerbi\_gateway\_instance\_count](#input\_powerbi\_gateway\_instance\_count) | Number of standalone Power BI On-Premises Gateway instances created directly from the launch template. | `number` | `2` | no |
 | <a name="input_private_subnet_cidrs"></a> [private\_subnet\_cidrs](#input\_private\_subnet\_cidrs) | List of CIDR blocks for private subnets. | `list(string)` | `[]` | no |
 | <a name="input_project"></a> [project](#input\_project) | The name of the Project we are bootstrapping tfscaffold for | `string` | n/a | yes |
 | <a name="input_public_subnet_cidrs"></a> [public\_subnet\_cidrs](#input\_public\_subnet\_cidrs) | List of CIDR blocks for public subnets. | `list(string)` | `[]` | no |
 | <a name="input_region"></a> [region](#input\_region) | The AWS Region | `string` | n/a | yes |
-| <a name="input_root_volume_size"></a> [root\_volume\_size](#input\_root\_volume\_size) | Size of root volume for the Power BI On-Premises Gateway instances - 30GB minimum for Windows Server | `number` | `30` | no |
+| <a name="input_root_volume_size"></a> [root\_volume\_size](#input\_root\_volume\_size) | Size of root volume for the Power BI On-Premises Gateway instances - 30GB minimum for Windows Server | `number` | `80` | no |
 | <a name="input_scale_in_recurrence_schedule"></a> [scale\_in\_recurrence\_schedule](#input\_scale\_in\_recurrence\_schedule) | The cron expression for the scale in schedule. Set to null if no recurrence is needed. | `string` | `null` | no |
 | <a name="input_scale_out_recurrence_schedule"></a> [scale\_out\_recurrence\_schedule](#input\_scale\_out\_recurrence\_schedule) | The cron expression for the scale out schedule. Set to null if no recurrence is needed. | `string` | `null` | no |
 | <a name="input_shared_infra_account_id"></a> [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Account ID of the shared infrastructure account | `string` | `"000000000000"` | no |

infrastructure/terraform/components/reporting/autoscaling_group_powerbi_gateway.tf

@@ -4,7 +4,7 @@ resource "aws_autoscaling_group" "powerbi_gateway" {
   name = local.csi
 
   launch_template {
-    id      = aws_launch_template.powerbi_gateway[0].id
+    id      = aws_launch_template.powerbi_gateway_asg[0].id
     version = "$Latest"
   }
 

infrastructure/terraform/components/reporting/cloudwatch_metric_alarm_patch_task_failed.tf

@@ -0,0 +1,18 @@
+resource "aws_cloudwatch_metric_alarm" "patch_task_failed" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  alarm_name          = "${local.csi}-patch-task-failed"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = "FailedCommands"
+  namespace           = "AWS/SSM-RunCommand"
+  period              = 300
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "Alarm when the AWS-RunPatchBaseline maintenance window task reports a failed run"
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    DocumentName = "AWS-RunPatchBaseline"
+  }
+}

infrastructure/terraform/components/reporting/cloudwatch_metric_alarm_powerbi_gateway_standalone_status.tf

@@ -0,0 +1,25 @@
+resource "aws_cloudwatch_metric_alarm" "powerbi_gateway_standalone_status_check_failed" {
+  for_each = var.enable_powerbi_gateway ? {
+    for idx, instance in aws_instance.powerbi_gateway_standalone :
+    idx => {
+      id   = instance.id
+      name = format("%s-powerbi-gateway-standalone-%02d-status-check-failed", local.csi, idx + 1)
+    }
+  } : {}
+
+  alarm_name          = each.value.name
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 2
+  datapoints_to_alarm = 2
+  metric_name         = "StatusCheckFailed"
+  namespace           = "AWS/EC2"
+  period              = 300
+  statistic           = "Maximum"
+  threshold           = 1
+  alarm_description   = "Instance or system status check failed for a standalone Power BI gateway host"
+  treat_missing_data  = "breaching"
+
+  dimensions = {
+    InstanceId = each.value.id
+  }
+}

infrastructure/terraform/components/reporting/ec2_instances_powerbi_gateway.tf

@@ -0,0 +1,13 @@
+# Standalone EC2 instances launched directly from the Power BI gateway launch template.
+resource "aws_instance" "powerbi_gateway_standalone" {
+  count = var.enable_powerbi_gateway ? var.powerbi_gateway_instance_count : 0
+
+  launch_template {
+    id      = aws_launch_template.powerbi_gateway_standalone[0].id
+    version = "$Latest"
+  }
+
+  tags = {
+    Name = format("%s-powerbi-gateway-standalone-%02d", local.csi, count.index + 1)
+  }
+}

infrastructure/terraform/components/reporting/launch_template_powerbi_gateway.tf → infrastructure/terraform/components/reporting/launch_template_powerbi_gateway_asg.tf

@@ -1,8 +1,8 @@
-resource "aws_launch_template" "powerbi_gateway" {
+resource "aws_launch_template" "powerbi_gateway_asg" {
   count = var.enable_powerbi_gateway ? 1 : 0
 
-  name                                 = local.csi
-  description                          = "Template for the Power BI On-Premises Gateway"
+  name                                 = "${local.csi}-asg"
+  description                          = "Template for the Power BI On-Premises Gateway (ASG)"
   update_default_version               = true
   image_id                             = "resolve:ssm:/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-Base"
   instance_type                        = var.instance_type

infrastructure/terraform/components/reporting/launch_template_powerbi_gateway_standalone.tf

@@ -0,0 +1,67 @@
+resource "aws_launch_template" "powerbi_gateway_standalone" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  name                                 = "${local.csi}-standalone"
+  description                          = "Template for the Power BI On-Premises Gateway (standalone instances)"
+  update_default_version               = true
+  image_id                             = "resolve:ssm:/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-Base"
+  instance_type                        = var.instance_type
+  user_data                            = data.cloudinit_config.powerbi_gateway[0].rendered
+  instance_initiated_shutdown_behavior = var.enable_spot ? "terminate" : "stop"
+  ebs_optimized                        = true
+
+  block_device_mappings {
+    device_name = "/dev/sda1"
+    ebs {
+      delete_on_termination = true
+      encrypted             = true
+      kms_key_id            = aws_kms_key.ebs[0].arn
+      volume_size           = var.root_volume_size
+      volume_type           = "gp3"
+    }
+  }
+
+  iam_instance_profile {
+    name = aws_iam_instance_profile.powerbi_gateway[0].name
+  }
+
+  dynamic "instance_market_options" {
+    for_each = var.enable_spot ? [1] : []
+    content {
+      market_type = "spot"
+      spot_options {
+        max_price          = var.spot_max_price
+        spot_instance_type = "one-time"
+      }
+    }
+  }
+
+  monitoring {
+    enabled = true
+  }
+
+  network_interfaces {
+    delete_on_termination       = true
+    associate_public_ip_address = false
+    security_groups = [
+      aws_security_group.powerbi_gateway[0].id
+    ]
+    subnet_id = element(module.powerbi_gateway_vpc[0].private_subnets, count.index)
+  }
+
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_tokens                 = "required"
+    http_put_response_hop_limit = 5
+  }
+
+  tag_specifications {
+    resource_type = "instance"
+    tags          = local.deployment_default_tags
+  }
+
+  tag_specifications {
+    resource_type = "volume"
+    tags          = local.deployment_default_tags
+  }
+}

infrastructure/terraform/components/reporting/ssm_maintenance_window_patch_window.tf

@@ -1,10 +1,21 @@
-resource "aws_ssm_maintenance_window" "patch_window" {
+resource "aws_ssm_maintenance_window" "patch_window_sunday" {
   count = var.enable_powerbi_gateway ? 1 : 0
 
-  name                       = "${local.csi}-windows-patch-window"
-  description                = "Windows Server 2022 Patch Window"
+  name                       = "${local.csi}-windows-patch-window-sun"
+  description                = "Windows Server 2022 Sunday Patch Window"
   schedule                   = "cron(0 3 ? * SUN *)" # Every Sunday at 3 AM
   duration                   = 4
   cutoff                     = 1
   allow_unassociated_targets = true
 }
+
+resource "aws_ssm_maintenance_window" "patch_window_wednesday" {
+  count = var.enable_powerbi_gateway ? 1 : 0
+
+  name                       = "${local.csi}-windows-patch-window-wed"
+  description                = "Windows Server 2022 Wednesday Patch Window"
+  schedule                   = "cron(0 3 ? * WED *)" # Every Wednesday at 3 AM
+  duration                   = 4
+  cutoff                     = 1
+  allow_unassociated_targets = true
+}

infrastructure/terraform/components/reporting/ssm_maintenance_window_target_windows_instances.tf

@@ -1,13 +1,41 @@
-resource "aws_ssm_maintenance_window_target" "windows_instances" {
+resource "aws_ssm_maintenance_window_target" "windows_instances_sunday_asg" {
   count = var.enable_powerbi_gateway ? 1 : 0
 
-  description   = "Windows Server 2022 Maintenance Window Target"
-  window_id     = aws_ssm_maintenance_window.patch_window[0].id
+  description   = "Windows Server 2022 Sunday Maintenance Window Target (ASG)"
+  window_id     = aws_ssm_maintenance_window.patch_window_sunday[0].id
   resource_type = "INSTANCE"
-  name          = "${local.csi}-maintenance-window-target"
+  name          = "${local.csi}-maintenance-window-target-sun-asg"
 
   targets {
     key    = "tag:Patch Group"
     values = ["${local.csi}-windows-group"]
   }
 }
+
+resource "aws_ssm_maintenance_window_target" "windows_instances_sunday_standalone" {
+  count = var.enable_powerbi_gateway && var.powerbi_gateway_instance_count >= 1 ? 1 : 0
+
+  description   = "Windows Server 2022 Sunday Maintenance Window Target (Standalone)"
+  window_id     = aws_ssm_maintenance_window.patch_window_sunday[0].id
+  resource_type = "INSTANCE"
+  name          = "${local.csi}-maintenance-window-target-sun-standalone"
+
+  targets {
+    key    = "InstanceIds"
+    values = [aws_instance.powerbi_gateway_standalone[0].id]
+  }
+}
+
+resource "aws_ssm_maintenance_window_target" "windows_instances_wednesday" {
+  count = var.enable_powerbi_gateway && var.powerbi_gateway_instance_count >= 2 ? 1 : 0
+
+  description   = "Windows Server 2022 Wednesday Maintenance Window Target"
+  window_id     = aws_ssm_maintenance_window.patch_window_wednesday[0].id
+  resource_type = "INSTANCE"
+  name          = "${local.csi}-maintenance-window-target-wed"
+
+  targets {
+    key    = "InstanceIds"
+    values = [aws_instance.powerbi_gateway_standalone[1].id]
+  }
+}

infrastructure/terraform/components/reporting/ssm_maintenance_window_task_patch_task.tf

@@ -1,27 +1,66 @@
-resource "aws_ssm_maintenance_window_task" "patch_task" {
+resource "aws_ssm_maintenance_window_task" "patch_task_sunday" {
   count = var.enable_powerbi_gateway ? 1 : 0
 
-  description = "Windows Server 2022 Patch Task"
-  window_id   = aws_ssm_maintenance_window.patch_window[0].id
+  description = "Windows Server 2022 Sunday Patch Task"
+  window_id   = aws_ssm_maintenance_window.patch_window_sunday[0].id
+  task_arn    = "AWS-RunPatchBaseline"
+  task_type   = "RUN_COMMAND"
+
+  targets {
+    key = "WindowTargetIds"
+    values = concat(
+      [aws_ssm_maintenance_window_target.windows_instances_sunday_asg[0].id],
+      var.powerbi_gateway_instance_count >= 1 ? [aws_ssm_maintenance_window_target.windows_instances_sunday_standalone[0].id] : []
+    )
+  }
+
+  task_invocation_parameters {
+    run_command_parameters {
+      comment = "Patching Sunday instances"
+      parameter {
+        name   = "Operation"
+        values = ["Install"]
+      }
+      parameter {
+        name   = "RebootOption"
+        values = ["RebootIfNeeded"]
+      }
+    }
+  }
+
+  priority        = 1
+  max_concurrency = "1"
+  max_errors      = "1"
+}
+
+resource "aws_ssm_maintenance_window_task" "patch_task_wednesday" {
+  count = var.enable_powerbi_gateway && var.powerbi_gateway_instance_count >= 2 ? 1 : 0
+
+  description = "Windows Server 2022 Wednesday Patch Task"
+  window_id   = aws_ssm_maintenance_window.patch_window_wednesday[0].id
   task_arn    = "AWS-RunPatchBaseline"
   task_type   = "RUN_COMMAND"
 
   targets {
     key    = "WindowTargetIds"
-    values = [aws_ssm_maintenance_window_target.windows_instances[0].id]
+    values = [aws_ssm_maintenance_window_target.windows_instances_wednesday[0].id]
   }
 
   task_invocation_parameters {
     run_command_parameters {
-      comment = "Patching Windows Instances"
+      comment = "Patching Wednesday instance"
       parameter {
         name   = "Operation"
         values = ["Install"]
       }
+      parameter {
+        name   = "RebootOption"
+        values = ["RebootIfNeeded"]
+      }
     }
   }
 
   priority        = 1
-  max_concurrency = "2"
+  max_concurrency = "1"
   max_errors      = "1"
 }

infrastructure/terraform/components/reporting/templates/cloudinit_config.tmpl

@@ -19,7 +19,7 @@ if (-not (Get-Command choco -ErrorAction SilentlyContinue)) {
 }
 
 # Install PowerBI On-Premises Gateway and Desktop
-choco install -y powerbigateway --version=3000.230.14 --ignore-checksums
+choco install -y powerbigateway --version=3000.298.8 --ignore-checksums
 choco install -y powerbi --ignore-checksums
 
 # Install vim
@@ -29,7 +29,7 @@ choco install -y vim
 choco install -y powershell-core
 
 # Install Amazon Athena ODBC 2.x Driver
-`$athenaDriverUrl = "https://s3.amazonaws.com/athena-downloads/drivers/ODBC/v2.0.3.0/Windows/AmazonAthenaODBC-2.0.3.0.msi"
+`$athenaDriverUrl = "https://downloads.athena.us-east-1.amazonaws.com/drivers/ODBC/v2.0.6.0/Windows/AmazonAthenaODBC-2.0.6.0.msi"
 `$athenaDriverInstaller = "C:\scripts\SimbaAthenaODBC.msi"
 Invoke-WebRequest -Uri `$athenaDriverUrl -OutFile `$athenaDriverInstaller
 

infrastructure/terraform/components/reporting/variables.tf

@@ -103,6 +103,12 @@ variable "enable_powerbi_gateway" {
   default     = true
 }
 
+variable "powerbi_gateway_instance_count" {
+  description = "Number of standalone Power BI On-Premises Gateway instances created directly from the launch template."
+  type        = number
+  default     = 2
+}
+
 variable "public_subnet_cidrs" {
   description = "List of CIDR blocks for public subnets."
   type        = list(string)
@@ -154,7 +160,7 @@ variable "spot_max_price" {
 variable "root_volume_size" {
   type        = number
   description = "Size of root volume for the Power BI On-Premises Gateway instances - 30GB minimum for Windows Server"
-  default     = 30
+  default     = 80
 }
 
 variable "scale_out_recurrence_schedule" {

infrastructure/terraform/etc/env_eu-west-2_int.tfvars

@@ -22,14 +22,6 @@ private_subnet_cidrs = [
   "10.0.6.0/24"
 ]
 
-instance_type    = "t3.medium"
-root_volume_size = 30
-desired_capacity = 1
-min_size         = 1
-max_size         = 1
-enable_spot      = false
-spot_max_price   = "0.3"
-
 enable_s3_backup = false
 
 shared_infra_account_id  = "099709604300"

infrastructure/terraform/etc/env_eu-west-2_main.tfvars

@@ -15,6 +15,10 @@ core_account_ids = [
 
 # PowerBI On-Premises Gateway variables:
 enable_powerbi_gateway = true
+min_size               = 2
+max_size               = 2
+desired_capacity       = 2
+
 
 public_subnet_cidrs = [
   "10.0.1.0/24",
@@ -28,14 +32,6 @@ private_subnet_cidrs = [
   "10.0.6.0/24"
 ]
 
-instance_type    = "t3.medium"
-root_volume_size = 30
-desired_capacity = 1
-min_size         = 1
-max_size         = 1
-enable_spot      = false
-spot_max_price   = "0.3"
-
 shared_infra_account_id  = "099709604300"
 
 destination_backup_vault_arn = "arn:aws:backup:eu-west-2:390844765011:backup-vault:nhs-notify-reporting-dev-backup-vault"