lsyncd · axkibe · Aug 22, 2017 · Aug 22, 2017 · Aug 22, 2017
diff --git a/examples/ls3.lua b/examples/ls3.lua
@@ -0,0 +1,222 @@
+----
+-- Example lsyncd configuration for syncing with an Amazon S3 bucket
+--
+-- This requires the official AWS CLI to be available, and that credentials
+-- bet set up through some external method, such as environment variables,
+-- IAM profiles or the AWS SDK configuration.
+--
+-- The AWS CLI sync exclude rules are not as powerful as the ones supported by
+-- lsyncd. Hence, some of the do not translate perfectly. For example, '*'
+-- (asterisk) matches slashes, while it does not in lsyncd. Hence it is a good
+-- idea to only use exclude patterns for full directories, either by using a
+-- trailing / (slash) or ** (double asterisk), as those will be correctly
+-- translated.
+--
+-- An initialSync options is provided as a convenience, since it's not easy to
+-- make sure exclusion rules match when doing it manually. It will *pull* from
+-- the target bucket to the local dir (the opposite of the regular behavior)
+-- then exit immediately.
+--
+-- Author: Daniel Miranda <danielkza2@gmail.com>
+--
+s3 = {}
+
+s3.checkgauge = {
+    >
+    >
+    >
+    >
+    >
+
+    delete      = true,
+    exclude     = true,
+    excludeFrom = true,
+    target      = true,
+
+    s3 = {
+        -- Path to the AWS CLI binary
+        awscliBinary  = true,
+        -- Extra options to pass to the AWS CLI (as a list)
+        awscliOptions = true,
+        -- Whether to do a dry-run, and not make any real changes
+        dryrun        = true,
+        -- Do an initial pull from the bucket and exit immediately.
+        initialSync   = true
+    }
+}
+
+-- Generate a list of exclude flags for the AWS CLI  based on the lsyncd
+-- patterns provided. Cache it to avoid re-generating it every time.
+
+local s3Excludes = function(config, excludes)
+    if config.s3._excludes == nil then
+        config.s3._excludes = {}
+        for _, pat in ipairs(excludes) do
+            pat = pat:gsub('%*%*', '[[ANY]]')
+            pat = pat:gsub('%?',   '[[ANY_BUT_SLASH_ONCE]]')
+            pat = pat:gsub('/$',   '/*')
+            pat = pat:gsub('%[%[ANY%]%]',                '*')
+            pat = pat:gsub('%[%[ANY_BUT_SLASH_ONCE%]%]', '[^/]')
+
+            if pat:match('^/') then
+                pat = pat:sub(2, -1)
+            else
+                pat = '*/' .. pat
+            end
+
+            table.insert(config.s3._excludes, '--exclude')
+            table.insert(config.s3._excludes, pat)
+        end
+
+        log('s3Excludes', table.concat(config.s3._excludes, '\n'))
+    end
+
+    return config.s3._excludes
+end
+
+-- Generates a command line to call the AWS CLI as configured, with the provided
+-- S3 action (such as cp, mv, rm or sync).
+-- Returns a tuple of (binaryPath, arguments)
+local awscliCommand = function(verb, config)
+    local bin = config.s3.awscliBinary
+    local args = {'s3', verb, '--only-show-errors'}
+    if config.s3.dryrun then
+        table.insert(args, '--dryrun')
+    end
+
+    if verb == 'sync'
+       and (config.delete == true or config.delete == 'startup')
+    then
+        table.insert(args, '--delete')
+    end
+
+    for _, opt in ipairs(config.s3.awscliOptions) do
+        table.insert(args, opt)
+    end
+
+    return bin, args
+end
+
+s3.action = function(inlet)
+    local event, event2 = inlet.getEvent()
+    -- S3 never actually deals with directories - they are just an illusion
+    -- created based on the common prefixes of objects. Hence discard any events
+    -- that do not concern files.
+    if event.isdir then
+        inlet.discardEvent(event)
+        return
+    end
+
+    local config = inlet.getConfig()
+    if event.etype == 'Create' or event.etype == 'Modify' then
+        local bin, args = awscliCommand('cp', config)
+        spawn(
+            event,
+            bin,
+            args,
+            event.sourcePath,
+            event.targetPath
+        )
+    elseif event.etype == 'Delete' then
+        if config.delete ~= true and config.delete ~= 'running' then
+            inlet.discardEvent(event)
+            return
+        end
+
+        local bin, args = awscliCommand('rm', config)
+        spawn(
+            event,
+            bin,
+            args,
+            event.targetPath
+        )
+    elseif event.etype == 'Move' then
+        local bin, args = awscliCommand('mv', config)
+        spawn(
+            event,
+            bin,
+            args,
+            event.targetPath,
+            event2.targetPath
+        )
+    else
+        log('Warn', 'ignored an event of type "', event.etype, '"')
+        inlet.discardEvent(event)
+    end
+end
+
+s3.init = function(event)
+    local config = event.config
+    local inlet = event.inlet
+    local excludes = s3Excludes(config, inlet.getExcludes())
+    local bin, args = awscliCommand('sync', config)
+
+    -- Do a pull when initialSync is enabled.
+    if config.s3.initialSync then
+        spawn(
+            event,
+            bin,
+            args,
+            excludes,
+            config.target,
+            event.sourcePath
+        )
+    -- And a push, as usual, otherwise
+    else
+        spawn(
+            event,
+            bin,
+            args,
+            excludes,
+            event.sourcePath,
+            config.target
+        )
+    end
+end
+
+-- Define a collect callback so we can terminate immediately when initialSync
+-- is enabled
+s3.collect = function(agent, exitcode)
+    local config = agent.config
+    if not agent.isList and agent.etype == 'Init' and config.s3.initialSync then
+        terminate(exitcode == 0 and 0 or -1)
+    end
+
+    return
+end
+
+s3.prepare = function(config, level)
+    default.prepare(config, level + 1)
+
+    config.target = config.target:gsub('/+$', '')
+    if not config.target:match('^s3://') then
+        config.target = 's3://' .. config.target
+    end
+end
+
+s3.s3 = {
+    awscliBinary  = '/usr/bin/aws',
+    awscliOptions = {},
+    dryrun       = false,
+    initialSync  = false
+}
+s3.delete       = false
+s3.delay        = 10
+s3.maxProcesses = 1
+
+sync {
+    s3,
+    source       = '/my/dir',
+    target       = 's3://my-bucket/my-path',
+    delay        = 30,
+    delete       = true,
+    maxProcesses = 2,
+    exclude = {
+        '/sub/folder/',
+    },
+    s3 = {
+        awscliBinary  = '/usr/local/bin/aws',
+        awscliOptions = {'--acl', 'public-read'},
+        dryrun        = false
+    }
+}