clean_backup/infrastructure/batch-infrastructure.yaml at main · NeotomaDB/clean_backup · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
AWSTemplateFormatVersion: '2010-09-09'
Description: 'AWS Batch infrastructure for Neotoma database sanitization. This infrastructure pulls a container from ECS (neotoma-sanitizer) and deploys it using AWS Batch.'

Parameters:
  VpcId:
    Type: AWS::EC2::VPC::Id
    Description: VPC where your RDS instance is located. No default here. This is private, and specific to Neotoma.
  SubnetIds:
    Type: List<AWS::EC2::Subnet::Id>
    Description: Private subnets with access to RDS
  DatabaseSecurityGroupId:
    Type: AWS::EC2::SecurityGroup::Id
    Description: Security group that has access to your RDS instance
  RemoteUser:
    Type: String
    NoEcho: true
    Description: Remote database username
  RemotePassword:
    Type: String
    NoEcho: true
    Description: Remote database password
  PostgresPassword:
    Type: String
    NoEcho: true
    Default: postgres
    Description: Local Postgres password
  ImageUri:
    Type: String
    Description: ECR image URI (from build-and-push.sh output)
  RDSEndpoint:
    Type: String
    Description: The address for the RDS server on which Neotoma is hosted.

Resources:
  # IAM Role for Batch Execution
  BatchExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: ecs-tasks.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy
      Policies:
        - PolicyName: S3Access
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - s3:PutObject
                  - s3:PutObjectAcl
                  - s3:GetObjectTagging
                Resource:
                  - arn:aws:s3:::neotoma-remote-store/*
              - Effect: Allow
                Action:
                  - s3:ListBucket
                Resource:
                  - arn:aws:s3:::neotoma-remote-store

  # Batch Compute Environment
  ComputeEnvironment:
    Type: AWS::Batch::ComputeEnvironment
    Properties:
      Type: MANAGED
      State: ENABLED
      ServiceRole: !Sub 'arn:aws:iam::${AWS::AccountId}:role/aws-service-role/batch.amazonaws.com/AWSServiceRoleForBatch'
      ComputeResources:
        Type: FARGATE
        MaxvCpus: 4
        Subnets: !Ref SubnetIds
        SecurityGroupIds:
          - !Ref DatabaseSecurityGroupId

  # Batch Job Queue
  JobQueue:
    Type: AWS::Batch::JobQueue
    Properties:
      State: ENABLED
      Priority: 1
      ComputeEnvironmentOrder:
        - Order: 1
          ComputeEnvironment: !Ref ComputeEnvironment

  # Job Definition
  JobDefinition:
    Type: AWS::Batch::JobDefinition
    Properties:
      Type: container
      PlatformCapabilities:
        - FARGATE
      ContainerProperties:
        Image: !Ref ImageUri
        EphemeralStorage:
          SizeInGiB: 50
        ResourceRequirements:
          - Type: VCPU
            Value: "2"
          - Type: MEMORY
            Value: "8192"
        JobRoleArn: !GetAtt BatchExecutionRole.Arn
        ExecutionRoleArn: !GetAtt BatchExecutionRole.Arn
        Environment:
          - Name: REMOTE_USER
            Value: !Ref RemoteUser
          - Name: REMOTE_PASSWORD
            Value: !Ref RemotePassword
          - Name: POSTGRES_PASSWORD
            Value: !Ref PostgresPassword
          - Name: RDS_ENDPOINT
            Value: !Ref RDSEndpoint
          - Name: RDS_PORT
            Value: "5432"
        NetworkConfiguration:
          AssignPublicIp: ENABLED
        FargatePlatformConfiguration:
          PlatformVersion: LATEST

  # EventBridge Rule for Monthly Execution
  ScheduleRule:
    Type: AWS::Events::Rule
    Properties:
      Description: "Run database sanitization monthly"
      ScheduleExpression: "cron(0 2 5 * ? *)"  # 2 AM on the 1st of every month
      State: ENABLED
      Targets:
        - Arn: !GetAtt JobQueue.JobQueueArn
          Id: "BatchJobTarget"
          RoleArn: !GetAtt EventBridgeRole.Arn
          BatchParameters:
            JobDefinition: !Ref JobDefinition
            JobName: "monthly-db-sanitization"

  # IAM Role for EventBridge
  EventBridgeRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: events.amazonaws.com
            Action: sts:AssumeRole
      Policies:
        - PolicyName: BatchJobExecution
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - batch:SubmitJob
                Resource: '*'

Outputs:
  JobDefinitionArn:
    Description: ARN of the Batch Job Definition
    Value: !Ref JobDefinition
  JobQueueArn:
    Description: ARN of the Batch Job Queue
    Value: !Ref JobQueue
  ComputeEnvironmentArn:
    Description: ARN of the Batch Compute Environment
    Value: !Ref ComputeEnvironment