node.js – 如何从Lambda函数解析AWS S3文件

我需要一些帮助来正确构建代码,以便使用S3存储桶和Lambda函数处理一些文本文件.

我想使用通过在S3存储桶中创建新对象而触发的Lambda函数来读取文件并提取一些数据并将其写入放置在另一个S3存储桶中的文件.

到目前为止,我的功能正常,将文件从一个S3存储桶复制到另一个存储桶,但我无法弄清楚如何添加一个函数来处理文件并将结果写入最终的S3目的地.

这些文件是简单的文本文件,我需要从文件中的每一行中提取数据.

如果我正在使用的Node.js代码添加了一个额外的函数来处理文件 – 请参阅注释？我在哪里寻求帮助.

// dependencies
var async = require('async');
var AWS = require('aws-sdk');
var util = require('util');


// get reference to S3 client 
var s3 = new AWS.S3();

exports.handler = function(event, context) {
    // Read options from the event.
    console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
    var srcBucket = event.Records[0].s3.bucket.name;
    // Object key may have spaces or unicode non-ASCII characters.
    var srcKey    =
    decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));  
    var dstBucket = "inputBucket";
    var dstKey    = srcKey + ".txt";

    // Sanity check: validate that source and destination are different buckets.
    if (srcBucket == dstBucket) {
        console.error("Destination bucket must not match source bucket.");
        return;
    }

    // Infer the file type.
    var typeMatch = srcKey.match(/\.([^.]*)$/);
    if (!typeMatch) {
        console.error('unable to infer file type for key ' + srcKey);
        return;
    }
    var imageType = typeMatch[1];
    if (imageType != "txt") {
        console.log('skipping non-image ' + srcKey);
        return;
    }

    // Download the image from S3, transform, and upload to a different S3 bucket.
    async.waterfall([
        function download(next) {
            // Download the file from S3 into a buffer.
            s3.getObject({
                    Bucket: srcBucket,
                    Key: srcKey
                },
                next);
            },
        function transform(response, next) {
            // Read the file we have just downloaded 
            // ? response.Body ?
            var rl = require('readline').createInterface({
                input: require('fs').createReadStream('file.in')
            });

            // Process each line here writing the result to an output buffer?
            rl.on('line', function (line) {
                 console.log('Line from file:', line);
                //Do something with the line... 

                //Create some output string 'outputline'

                //Write 'outputline' to an output buffer 'outbuff'
                // ??

            });
            // Now pass the output buffer to the next function
            // so it can be uploaded to another S3 bucket 
            // ?? 
            next;
        }
        function upload(response, next) {
            // Stream the file to a different S3 bucket.
            s3.putObject({
                    Bucket: dstBucket,
                    Key: dstKey,
                    Body: response.Body,
                    ContentType: response.contentType
                },
                next);
            }
        ], function (err) {
            if (err) {
                console.error(
                    'Unable to process ' + srcBucket + '/' + srcKey +
                    ' and upload to ' + dstBucket + '/' + dstKey +
                    ' due to an error: ' + err
                );
            } else {
                console.log(
                    'Successfully processed ' + srcBucket + '/' + srcKey +
                    ' and uploaded to ' + dstBucket + '/' + dstKey
                );
            }

            context.done();
        }
    );
};

在s3.getObject的回调中

s3.getObject(params,function(err,data){})

如果您的文件是文本,那么您可以将文本提取为字符串

data.Body.toString("utf-8")