Commit daf67988 authored by Milica Zivkov's avatar Milica Zivkov

Lambda + terraform

parents
### Terraform ###
# Local .terraform directories
**/.terraform/*
# .tfstate files
*.tfstate
*.tfstate.*
# Crash log files
crash.log
# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
# .tfvars files are managed as part of configuration and so should be included in
# version control.
#
# example.tfvars
### IntelliJ ###
/out/
.idea_modules/
.idea/
*.iml
# Dependency directories
node_modules/
jspm_packages/
*zip
dist
# should be secret
# ftp-configuration.json
# output
deployment/
This diff is collapsed.
{
"name": "move-ftp-files-to-s3",
"version": "1.0.0",
"description": "AWS Lambda source for downloading files from FTP to S3.",
"author": "",
"license": "ISC",
"scripts": {
"build:for:deployment": "npm run build-ts && rm -Rf node_modules && npm install --only prod",
"build-ts": "rm -Rf node_modules && npm install && rm -Rf dist && tsc"
},
"devDependencies": {
"aws-sdk": "latest",
"@types/aws-lambda": "^8.10.11",
"@types/node": "^8.10.11",
"@types/aws-sdk": "^2.7.0",
"ts-node": "^7.0.1",
"typescript": "3.0.3",
"tslint": "^5.11.0",
"tslint-config-airbnb-base": "^0.2.0"
},
"dependencies": {
"csvtojson": "^2.0.8",
"ftp": "^0.3.10"
}
}
import { Handler } from 'aws-lambda';
import AWS from 'aws-sdk';
import { MoveFtpFilesToS3Lambda } from './moveFtpFilesToS3Lambda';
import { S3Storage } from './s3/s3Storage';
import { FtpClient } from './ftp/ftpClient';
import { SsmClient } from './ssm/ssmClient';
import { ImportFilesEvent } from './importFilesEvent';
const s3 = new AWS.S3();
const s3Storage = new S3Storage(s3);
const ssm = new AWS.SSM();
const ssmClient = new SsmClient(ssm);
const ftpClient = new FtpClient();
const handler: Handler<ImportFilesEvent, void> = async (event: ImportFilesEvent) => {
console.log(`start execution for event ${JSON.stringify(event)}`);
try {
const lambda = new MoveFtpFilesToS3Lambda(process.env, ssmClient, ftpClient, s3Storage);
await lambda.execute(event);
} catch (e) {
console.log(e);
throw new Error(e);
}
};
export { handler };
import { FtpConfig } from './ftpConfig';
import { FtpFileDesc } from './ftpFileDesc';
var Client = require('ftp');
class FtpClient {
private config: FtpConfig;
private client;
configure(jsonConfig: string) {
try {
this.config = this.getValidatedConfig(jsonConfig);
this.config.connTimeout = 300;
} catch (e) {
console.log(e);
throw Error('FTP Configuration is not valid');
}
}
private getValidatedConfig(config: string): FtpConfig {
const json = JSON.parse(config);
const configPrototype: FtpConfig = {
host: '',
port: 0,
user: '',
password: ''
};
for (const key of Object.keys(configPrototype)) {
if (!json.hasOwnProperty(key)) {
throw new Error(`JSON Body does not have required property: ${key}`);
}
}
return json;
}
connect(): Promise<void> {
return new Promise((resolve, reject) => {
this.client = new Client();
this.client.on('ready', () => {
console.log('FTP Connection successful');
resolve();
});
this.client.on('error', (error) => {
console.log(error);
reject(error);
});
this.client.connect(this.config);
});
}
list(path: string): Promise<FtpFileDesc[]> {
return new Promise((resolve, reject) => {
this.client.list(path, (error, list) => {
if (error) {
reject(error);
} else {
resolve(list);
}
});
});
}
get(path: string): Promise<any> {
return new Promise((resolve, reject) => {
this.client.get(path, (error, stream) => {
if (error) {
reject(error);
} else {
let file = '';
stream.on('data', (chunk) => {
file += chunk;
});
stream.on('end', () => {
resolve(file);
});
stream.on('error', (error) => {
reject(error);
});
}
});
});
}
delete(path: string): Promise<void> {
return new Promise((resolve, reject) => {
this.client.delete(path, (error) => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
}
disconnect() {
this.client.end();
}
}
export { FtpClient };
interface FtpConfig {
host: string;
port: number;
user: string;
password: string;
connTimeout?: number;
}
export { FtpConfig };
interface FtpFileDesc {
name: string;
type: string;
size: number;
date: Date;
}
export { FtpFileDesc };
\ No newline at end of file
interface ImportFilesEvent {
ftp_path: string;
s3_bucket: string;
}
export { ImportFilesEvent };
import ProcessEnv = NodeJS.ProcessEnv;
import { FtpClient } from './ftp/ftpClient';
import { S3Storage } from './s3/s3Storage';
import { SsmClient } from './ssm/ssmClient';
import { ImportFilesEvent } from './importFilesEvent';
class MoveFtpFilesToS3Lambda {
private readonly ftpConfigParameter: string;
constructor(env: ProcessEnv, public ssm: SsmClient, public ftp: FtpClient, public s3: S3Storage) {
this.ftpConfigParameter = this.getFtpConfigParameter(env);
}
private getFtpConfigParameter(env: ProcessEnv): string {
if (!env.FTP_CONFIG_PARAMETER) {
throw new Error('env variable FTP_CONFIG_PARAMETER must be defined');
}
return env.FTP_CONFIG_PARAMETER;
}
async execute(event: ImportFilesEvent): Promise<void> {
const ftpConfig = await this.readFtpConfiguration();
this.ftp.configure(ftpConfig);
await this.ftp.connect();
const files = await this.ftp.list(event.ftp_path);
for (const ftpFile of files) {
const fileStream = await this.ftp.get(`${event.ftp_path}/${ftpFile.name}`);
await this.s3.put(fileStream, ftpFile.name, event.s3_bucket);
await this.ftp.delete(`${event.ftp_path}/${ftpFile.name}`);
}
this.ftp.disconnect();
}
private async readFtpConfiguration(): Promise<string> {
try {
return await this.ssm.getParameter(this.ftpConfigParameter);
} catch (e) {
console.log(e);
throw Error(`AWS Parameter Store doesn't have ${this.ftpConfigParameter} parameter created`);
}
}
}
export { MoveFtpFilesToS3Lambda };
import S3 = require('aws-sdk/clients/s3');
import ReadableStream = NodeJS.ReadableStream;
class S3Storage {
private s3: S3;
constructor(s3: S3) {
this.s3 = s3;
}
async put(object: ReadableStream, objectKey: string, bucket: string): Promise<any> {
const putRequest = {
Body: object,
Bucket: bucket,
Key: objectKey
};
return await this.s3.putObject(putRequest).promise();
}
}
export { S3Storage };
import SSM = require('aws-sdk/clients/ssm');
import { PSParameterValue } from 'aws-sdk/clients/ssm';
class SsmClient {
constructor(public ssm:SSM){
}
async getParameter(param: string): Promise<PSParameterValue> {
const ftpConfig = await this.ssm.getParameter({Name: param, WithDecryption: true}).promise();
return ftpConfig.Parameter.Value;
}
}
export { SsmClient };
{
"compileOnSave": false,
"compilerOptions": {
"outDir": "dist",
"baseUrl": ".",
"sourceMap": true,
"declaration": false,
"module": "commonjs",
"moduleResolution": "node",
"esModuleInterop": true,
"noImplicitAny": false,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"target": "es6",
"paths": {
"*": [
"node_modules/*",
"src/types/*"
]
},
"lib": ["es6", "esnext"]
},
"include": [
"src/**/*"
]
}
{
"extends": "tslint-config-airbnb-base",
"rules": {
"ter-max-len": {
"options": [120]
},
"no-console": false,
"ter-indent": [true, 4],
"ter-padded-blocks": [true, { "blocks": "never", "classes": "always", "switches": "never" }],
"trailing-comma": false,
"no-use-before-declare": false
}
}
# Providing Credentials for FTP server
Credentials will be provided by CI Pipeline. Do not commit credentials to Git.
{
"host": "set_host",
"port": 21,
"user": "set_user",
"password": "set_password"
}
\ No newline at end of file
resource "aws_ssm_parameter" "ftp_config" {
name = "ftp_config"
description = "FTP Server Configuration used in ftp-to-s3-transfer lambda"
type = "SecureString"
value = "${data.template_file.ftp_config_json.rendered}"
}
data "template_file" "ftp_config_json" {
template = "${file("${path.module}/credentials/ftp-configuration.json")}"
}
\ No newline at end of file
module "move_ftp_files_to_s3_lambda" {
source = "./modules/aws-lambda"
function_name = "move-ftp-files-to-s3"
description = "Move files from configured FTP server and folder into configured S3 Bucket/path"
handler = "entrypoint.handler"
runtime = "nodejs8.10"
code_version = "v1.0.0"
code_bucket = "${aws_s3_bucket.lambda_code_bucket.bucket}"
environment_variables = {
"FTP_CONFIG_PARAMETER" = "${aws_ssm_parameter.ftp_config.name}",
}
}
# Allow Lambda to write to any bucket within this AWS Account
resource "aws_iam_policy_attachment" "lambda_iam_policy_attachment" {
name = "${module.move_ftp_files_to_s3_lambda.function_name}-policy"
roles = ["${module.move_ftp_files_to_s3_lambda.role_name}"]
policy_arn = "${aws_iam_policy.lambda_iam_policy.arn}"
}
resource "aws_iam_policy" "lambda_iam_policy" {
name = "${module.move_ftp_files_to_s3_lambda.function_name}-policy"
policy = "${data.aws_iam_policy_document.lambda_iam_policy_document.json}"
}
data "aws_iam_policy_document" "lambda_iam_policy_document" {
statement {
effect = "Allow"
actions = [
"s3:PutObject",
"s3:PutObjectAcl"
]
resources = [
"arn:aws:s3:::${aws_s3_bucket.destination_one_bucket.bucket}/*",
"arn:aws:s3:::${aws_s3_bucket.destination_two_bucket.bucket}/*"
]
},
statement {
effect = "Allow"
actions = [
"ssm:GetParameter"
]
resources = [
"arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/*"
]
}
}
\ No newline at end of file
provider "aws" {
region = "eu-central-1"
version = "~> 1.30"
}
data "aws_region" "current" {}
data "aws_caller_identity" "current" {}
\ No newline at end of file
data "aws_region" "current" {}
data "aws_caller_identity" "current" {}
resource "aws_lambda_function" "lambda" {
function_name = "${var.function_name}"
description = "${var.description}"
handler = "${var.handler}"
runtime = "${var.runtime}"
memory_size = "${var.memory_size}"
timeout = "${var.timeout}"
s3_bucket = "${var.code_bucket}"
s3_key = "${var.package_name}"
role = "${aws_iam_role.lambda_basic_exec_role.arn}"
publish = true
environment {
variables = "${var.environment_variables}"
}
}
resource "aws_iam_role" "lambda_basic_exec_role" {
name = "${var.function_name}_role"
assume_role_policy = "${data.aws_iam_policy_document.assume_role_policy_document.json}"
}
data "aws_iam_policy_document" "assume_role_policy_document" {
statement {
effect = "Allow"
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["lambda.amazonaws.com"]
}
}
}
resource "aws_iam_policy_attachment" "logs_policy_attachment" {
name = "${var.function_name}-logs"
roles = ["${aws_iam_role.lambda_basic_exec_role.name}"]
policy_arn = "${aws_iam_policy.logs_policy.arn}"
}
resource "aws_iam_policy" "logs_policy" {
name = "${var.function_name}-logs"
policy = "${data.aws_iam_policy_document.logs_policy_document.json}"
}
data "aws_iam_policy_document" "logs_policy_document" {
statement {
effect = "Allow"
actions = [
"logs:CreateLogGroup",
]
resources = [
"arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*",
]
}
statement {
effect = "Allow"
actions = [
"logs:CreateLogStream",
"logs:PutLogEvents",
]
resources = [
"arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/${var.function_name}:*",
]
}
}
\ No newline at end of file
output "function_arn" {
value = "${aws_lambda_function.lambda.arn}"
}
output "function_name" {
value = "${aws_lambda_function.lambda.function_name}"
}
output "role_name" {
value = "${aws_iam_role.lambda_basic_exec_role.name}"
}
\ No newline at end of file
variable "function_name" {
type = "string"
}
variable "description" {
type = "string"
}
variable "memory_size" {
type = "string"
default = 1024
}
variable "timeout" {
type = "string"
default = "3"
}
variable "runtime" {
type = "string"
}
variable "handler" {
type = "string"
}
variable "code_version" {
type = "string"
}
variable "package_name" {
type = "string"
default = "Lambda-Deployment.zip"
}
variable "code_bucket" {
type = "string"
}
variable "environment_variables" {
type = "map"
default = {
"default_variable" = "default_value"
}
}
\ No newline at end of file
resource "aws_s3_bucket" "lambda_code_bucket" {
bucket = "move-ftp-files-to-s3-code"
}
resource "aws_s3_bucket" "destination_one_bucket" {
bucket = "destination-one"
}
resource "aws_s3_bucket" "destination_two_bucket" {
bucket = "destination-two"
}
resource "aws_cloudwatch_event_rule" "source_one_event_rule" {
name = "source-one-event"
description = "Fires every five minutes"
schedule_expression = "cron(0/5 * * * ? *)"
}
resource "aws_cloudwatch_event_target" "move_files_to_destination_one" {
rule = "${aws_cloudwatch_event_rule.source_one_event_rule.name}"
target_id = "move-files-to-destination-one"
arn = "${module.move_ftp_files_to_s3_lambda.function_arn}"
input = "${data.template_file.source_destination_one_config.rendered}"
}
data "template_file" "source_destination_one_config" {
template = <<DOC
{
"ftp_path": "$${ftp_path}",
"s3_bucket": "$${bucket_name}"
}
DOC
vars {
ftp_path = "source1"
bucket_name = "${aws_s3_bucket.destination_one_bucket.bucket}"
}
}
resource "aws_lambda_permission" "source_one_invokes_lambda" {
statement_id = "source_one_invokes_lambda"
action = "lambda:InvokeFunction"