Photo by Johannes Plenio on Unsplash

Building a CI/CD pipeline for an AWS Lambda function using AWS CodePipeline

Introduction

What is CI/CD?

Continuous Integration

Continuous Delivery

Continuous Deployment

AWS services for CI/CD

AWS CloudFormation

AWS Serverless Application Model (AWS SAM)

Building a CI/CD pipeline

The serverless application

The SAM template

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Parameters:
TargetEmail:
Type: String
Resources:
SourceBucket:
Type: AWS::S3::Bucket
DeletionPolicy: Retain
Properties:
BucketName: !Sub "demo-ci-cd-source-bucket"
TargetTopic:
Type: AWS::SNS::Topic
Properties:
TopicName: demo-ci-cd-target-topic
Subscription:
- Endpoint: !Sub "${TargetEmail}"
Protocol: email
S3ToSNSFunction:
Type: AWS::Serverless::Function
Properties:
FunctionName: demo-ci-cd-lambda-function
Handler: demo.S3ToSnsHandler::handleRequest
Runtime: java11
CodeUri: build/distributions/demo-ci-cd-lambda-function.zip
Environment:
Variables:
Region: !Sub "${AWS::Region}"
TopicARN: !Sub "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:demo-ci-cd-target-topic"
AutoPublishAlias: live
DeploymentPreference:
Type: AllAtOnce
Timeout: 60
MemorySize: 512
Policies:
- LambdaInvokePolicy:
FunctionName: demo-ci-cd-lambda-function
- S3ReadPolicy:
BucketName: !Sub "demo-ci-cd-source-bucket"
- SNSPublishMessagePolicy:
TopicName: demo-ci-cd-target-topic
Events:
S3ObjectCreated:
Type: S3
Properties:
Bucket: !Ref SourceBucket
Events: s3:ObjectCreated:*
S3ObjectRemoved:
Type: S3
Properties:
Bucket: !Ref SourceBucket
Events: s3:ObjectRemoved:*

The Lambda function

public class S3ToSnsHandler implements RequestHandler<S3Event, Void> {   @Override
public Void handleRequest(S3Event event, Context context) {
List<String> messages = new ArrayList<>();
for (S3EventNotificationRecord record: event.getRecords()) {
String eventName = record.getEventName();
String bucketName = record.getS3().getBucket().getName();
String objectKey = record.getS3().getObject().getKey();
if ("ObjectCreated:Put".equals(eventName)) {
messages.add(String.format("Object %s is created in bucket %s",
objectKey, bucketName));
}
if ("ObjectRemoved:Delete".equals(eventName)) {
messages.add(String.format("Object %s is removed from bucket %s",
objectKey, bucketName));
}
}
String body = String.join("\n", messages); String region = System.getenv("Region");
String topicARN = System.getenv("TopicARN");
PublishRequest publishRequest = new PublishRequest(topicARN, body);
PublishResult publishResult = getAmazonSNS(region).publish(publishRequest);
return null;
}
AmazonSNS getAmazonSNS(String region) {
return AmazonSNSClient.builder()
.withRegion(Regions.fromName(region))
.withCredentials(new DefaultAWSCredentialsProviderChain())
.build();
}
}
public class S3ToSnsHandlerTest {   @ParameterizedTest
@MethodSource("methodSource")
public void handleRequestTest(String fileName, String message) throws Exception {
URL fileURL = getClass().getClassLoader().getResource(fileName);
String json = Files.readString(Path.of(fileURL.toURI()), StandardCharsets.UTF_8);
S3Event event = new S3Event(S3Event.parseJson(json).getRecords());
Map<String, String> environment = new HashMap<>();
environment.put("Region", "eu-north-1");
environment.put("TopicARN", "arn:aws:sns:::target-topic");
setEnvironment(environment);
Context context = createMock(Context.class);
AmazonSNS amazonSNS = createMock(AmazonSNS.class);
S3ToSnsHandler handler = new S3ToSnsHandler() {
@Override
AmazonSNS getAmazonSNS(String region) {
assertEquals("eu-north-1", region);
return amazonSNS;
}
};
Capture<PublishRequest> publishRequestCapture = newCapture();
PublishResult publishResult = createMock(PublishResult.class);
expect(amazonSNS.publish(capture(publishRequestCapture))).andReturn(publishResult).once(); replay(context, amazonSNS, publishResult);
handler.handleRequest(event, context);
verify(context, amazonSNS, publishResult);
PublishRequest publishRequest = publishRequestCapture.getValue();
assertEquals(message, publishRequest.getMessage());
assertEquals("arn:aws:sns:::target-topic", publishRequest.getTopicArn());
}
private static Stream<Arguments> methodSource() {
return Stream.of(
Arguments.of("ObjectCreatedPut.json", "Object test/key is created in bucket source-bucket"),
Arguments.of("ObjectRemovedDelete.json", "Object test/key is removed from bucket source-bucket")
);
}
}

The build specification

version: 0.2
phases:
install:
runtime-versions:
java: corretto11
build:
commands:
- ./gradlew clean buildZip
- sam package
--template-file template.yml
--output-template-file package.yml
--s3-bucket demo-ci-cd-sam-bucket
artifacts:
files:
- package.yml

The pipeline

Repository

git update-index --chmod=+x gradlew

Pipeline settings

Source stage

Build

Build stage

Service role for CloudFormation

{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": [
"cloudformation:CreateChangeSet",
"codedeploy:CreateApplication",
"codedeploy:CreateDeployment",
"codedeploy:CreateDeploymentGroup",
"codedeploy:DeleteApplication",
"codedeploy:DeleteDeploymentGroup",
"codedeploy:GetDeployment",
"codedeploy:GetDeploymentConfig",
"codedeploy:RegisterApplicationRevision",
"iam:AttachRolePolicy",
"iam:CreateRole",
"iam:DeleteRole",
"iam:DeleteRolePolicy",
"iam:DetachRolePolicy",
"iam:GetRole",
"iam:GetRolePolicy",
"iam:PassRole",
"iam:PutRolePolicy",
"iam:TagRole",
"iam:UntagRole",
"lambda:AddPermission",
"lambda:CreateAlias",
"lambda:CreateFunction",
"lambda:DeleteAlias",
"lambda:DeleteFunction",
"lambda:GetAlias",
"lambda:GetFunction",
"lambda:ListVersionsByFunction",
"lambda:PublishVersion",
"lambda:RemovePermission",
"lambda:UpdateFunctionCode",
"lambda:UpdateFunctionConfiguration",
"s3:CreateBucket",
"s3:DeleteBucket",
"s3:GetObject",
"s3:PutBucketNotification",
"sns:CreateTopic",
"sns:DeleteTopic",
"sns:GetTopicAttributes",
"sns:Publish",
"sns:Subscribe",
"sns:Unsubscribe"
],
"Resource": "*"
}
]
}

Deploy stage

Review

Running a CI/CD pipeline

Test the Lambda function

sam local generate-event s3 put --bucket source-bucket --key test/key | sam local invoke -e - demo-ci-cd-lambda-function
Reading invoke payload from stdin (you can also pass it from file with --event)
Invoking demo.S3ToSnsHandler::handleRequest (java11)
Decompressing D:\GitHub\demo-ci-cd-lambda-function\build\distributions\demo-ci-cd-lambda-function.zip
Skip pulling image and use local one: amazon/aws-sam-cli-emulation-image-java11:rapid-1.26.0.
Mounting C:\Users\John_Doe\AppData\Local\Temp\tmpnqhgkgpe as /var/task:ro,delegated inside runtime container
START RequestId: 56086afc-1a57-4ed2-b67a-108694a8384d Version: $LATEST
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - S3 event: ...
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - context: ...
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - S3 message: ...
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - event name: ObjectCreated:Put
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - S3 bucket: source-bucket
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - S3 key: test/key
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - SNS message body: "Object test/key is created in bucket source-bucket"
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - region: eu-north-1
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - topic ARN: arn:aws:sns:eu-north-1:798059032812:demo-ci-cd-target-topic
2021-08-20 15:59:46 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - SNS publish request: ...
2021-08-20 15:59:49 56086afc-1a57-4ed2-b67a-108694a8384d INFO S3ToSnsHandler - SNS publish result: ...
END RequestId: 56086afc-1a57-4ed2-b67a-108694a8384d
REPORT RequestId: 56086afc-1a57-4ed2-b67a-108694a8384d Init Duration: 0.20 ms Duration: 4942.83 ms Billed Duration: 5000 ms Memory Size: 512 MB Max Memory Used: 512 MB
null

Run the serverless application

Modify the serverless application

Conclusion

Senior Software Engineer