2

Trying to simply deploy a data pipeline definition which worked before. This time I have changed roles to be consistent with another AWS (production vs staging) account.

When I deploy via AWS CLI, I get this message:

{
    "validationErrors": [
        {
            "errors": [
                "Please add following permissions to the role ('DataPipelineDefaultRole') for uploading logs to s3: s3:Get*,s3:List*,s3:Put*"
            ], 
            "id": "EC2_Box_TaskRunner"
        }
    ], 
    "errored": true, 
    "validationWarnings": []
}

Here is definition of DataPipelineDefaultRole:

{                                                                                                                                                                                          
    "Statement": [                                                                                                                                                                         
        {                                                                                                                                                                                  
            "Action": [                                                                                                                                                                    
                "s3:*",                                                                                                                                                         
                "dynamodb:DescribeTable",                                                                                                                                                  
                "dynamodb:Scan",                                                                                                                                                           
                "dynamodb:Query",                                                                                                                                                          
                "dynamodb:GetItem",                                                                                                                                                        
                "dynamodb:BatchGetItem",                                                                                                                                                   
                "dynamodb:UpdateTable",

                "ec2:*",

                "elasticmapreduce:*",                                                                                                                                                      
                "rds:DescribeDBInstances",                                                                                                                                                 
                "rds:DescribeDBSecurityGroups",                                                                                                                                            
                "redshift:DescribeClusters",                                                                                                                                               
                "redshift:DescribeClusterSecurityGroups",                                                                                                                                  
                "sns:GetTopicAttributes",                                                                                                                                                  
                "sns:ListTopics",                                                                                                                                                          
                "sns:Publish",                                                                                                                                                             
                "sns:Subscribe",                                                                                                                                                           
                "sns:Unsubscribe",                                                                                                                                                         
                "iam:PassRole",                                                                                                                                                            
                "iam:ListRolePolicies",                                                                                                                                                    
                "iam:GetRole",                                                                                                                                                             
                "iam:GetRolePolicy",                                                                                                                                                       
                "iam:ListInstanceProfiles",                                                                                                                                                
                "cloudwatch:*",                                                                                                                                                            
                "datapipeline:*"                                                                                                                                          
            ],                                                                                                                                                                             
            "Effect": "Allow",                                                                                                                                                             
            "Resource": [                                                                                                                                                                  
                "*"                                                                                                                                                                        
            ]                                                                                                                                                                              
        }                                                                                                                                                                                  
    ]                                                                                                                                                                                      
}

I also noticed that when I tried to recreate the 'DataPipelineDefaultRole', sometimes the "Instance Profile ARN(s)" is blank. Finally I was able to recreate the role - through combination of "hello world data pipeline template" setups and/or manual steps. (with Instance ARN present)

Even with role set as:

{                                                                                                                                                                                          
    "Statement": [                                                                                                                                                                         
        {                                                                                                                                                                                  
            "Action": [                                                                                                                                                                    
                "*"                                                                                                                                          
            ],                                                                                                                                                                             
            "Effect": "Allow",                                                                                                                                                             
            "Resource": [                                                                                                                                                                  
                "*"                                                                                                                                                                        
            ]                                                                                                                                                                              
        }                                                                                                                                                                                  
    ]                                                                                                                                                                                      
}

it does not work.

What am I doing wrong?

1 Answer 1

0

Ok the problem was with s3 paths referenced in other objects in pipeline definition, not the role or EC2_Box_TaskRunner box

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.