From b684bf2ca520f5ff9a420e46d4481635edc4923d Mon Sep 17 00:00:00 2001
From: David Kebler <d@kebler.net>
Date: Fri, 8 Feb 2019 22:27:44 -0800
Subject: [PATCH] working sync module extending the node-rsync module

---
 .eslintrc.js                  |   37 ++
 .gitignore                    |    2 +
 .npmignore                    |    6 +
 nodemon.json                  |    3 +
 package.json                  |   51 ++
 src/rsync.js                  | 1115 +++++++++++++++++++++++++++++++++
 src/sync.js                   |  195 ++++++
 src/watcher.js                |   41 ++
 test/jobs/local.yaml          |    8 +
 test/jobs/options/mirror.yaml |   20 +
 test/jobs/sample.yaml         |   12 +
 test/jobs/ssh/switches.yaml   |    7 +
 test/repo/.gitignore          |    2 +
 test/repo/.npmignore          |    5 +
 test/sync.test.js             |   58 ++
 15 files changed, 1562 insertions(+)
 create mode 100644 .eslintrc.js
 create mode 100644 .gitignore
 create mode 100644 .npmignore
 create mode 100644 nodemon.json
 create mode 100644 package.json
 create mode 100644 src/rsync.js
 create mode 100644 src/sync.js
 create mode 100644 src/watcher.js
 create mode 100644 test/jobs/local.yaml
 create mode 100644 test/jobs/options/mirror.yaml
 create mode 100644 test/jobs/sample.yaml
 create mode 100644 test/jobs/ssh/switches.yaml
 create mode 100644 test/repo/.gitignore
 create mode 100644 test/repo/.npmignore
 create mode 100644 test/sync.test.js

diff --git a/.eslintrc.js b/.eslintrc.js
new file mode 100644
index 0000000..49bac18
--- /dev/null
+++ b/.eslintrc.js
@@ -0,0 +1,37 @@
+module.exports = {
+  "ecmaFeatures": {
+    "modules": true,
+    "spread" : true,
+    "restParams" : true
+  },
+  // "plugins": [
+  // "unicorn"
+  //  ],
+  "env": {
+    "es6": true,
+    "node": true,
+    "mocha": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 2017,
+    "sourceType": "module"
+  },
+  "extends": "eslint:recommended",
+  "rules": {
+    "indent": [
+      "error",
+      2
+    ],
+    // "unicorn/no-array-instanceof": "error",
+    "no-console": 0,
+    "semi": ["error", "never"],
+    "linebreak-style": [
+      "error",
+      "unix"
+    ],
+    "quotes": [
+      "error",
+      "single"
+    ]
+  }
+}
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..e61051f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+/node_modules/
+/coverage/
diff --git a/.npmignore b/.npmignore
new file mode 100644
index 0000000..90e0970
--- /dev/null
+++ b/.npmignore
@@ -0,0 +1,6 @@
+tests/
+test/
+*.test.js
+testing/
+example/
+nodemon.json
diff --git a/nodemon.json b/nodemon.json
new file mode 100644
index 0000000..de7e448
--- /dev/null
+++ b/nodemon.json
@@ -0,0 +1,3 @@
+{
+  "ext": "js, yaml"
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..854d7e9
--- /dev/null
+++ b/package.json
@@ -0,0 +1,51 @@
+{
+  "name": "@uci/remote-code",
+  "version": "0.0.2",
+  "description": "module to copy, maintain, and launch hardware modules on other machines",
+  "main": "src/index.js",
+  "bin": {
+    "ssh": "./bin/ssh.js"
+  },
+  "scripts": {
+    "sync": "node -r esm ./bin/ssh",
+    "testd": "UCI_ENV=dev ./node_modules/.bin/nodemon --exec './node_modules/.bin/mocha -r esm --timeout 30000'",
+    "test": "UCI_ENV=pro UCI_LOG_PATH=./test/test.log ./node_modules/.bin/mocha -r esm --timeout 30000 || exit 0",
+    "testibc": "istanbul cover ./node_modules/.bin/_mocha test/ --report lcovonly -- -R spec --recursive && codecov || true"
+  },
+  "author": "David Kebler",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/uCOMmandIt/uci-remote-code.git"
+  },
+  "keywords": [
+    "node.js",
+    "I2C",
+    "raspberryPi"
+  ],
+  "bugs": {
+    "url": "https://github.com/uCOMmandIt/uci-remote-code/issues"
+  },
+  "homepage": "https://github.com/uCOMmandIt/uci-remote-code#readme",
+  "dependencies": {
+    "@uci/logger": "0.0.10",
+    "aggregation": "^1.2.5",
+    "await-to-js": "^2.1.1",
+    "chokidar": "^2.0.4",
+    "conf": "^2.2.0",
+    "esm": "^3.1.4",
+    "fs-read-data": "^1.0.4",
+    "globby": "^9.0.0",
+    "p-settle": "^2.1.0",
+    "path-exists": "^3.0.0",
+    "yargs": "^12.0.5"
+  },
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "chai-arrays": "^2.0.0",
+    "codecov": "^3.1.0",
+    "istanbul": "^0.4.5",
+    "mocha": "^5.x",
+    "nodemon": "^1.18.9"
+  }
+}
diff --git a/src/rsync.js b/src/rsync.js
new file mode 100644
index 0000000..0d20244
--- /dev/null
+++ b/src/rsync.js
@@ -0,0 +1,1115 @@
+var spawn = require('child_process').spawn
+var path = require('path')
+
+/**
+ * Rsync is a wrapper class to configure and execute an `rsync` command
+ * in a fluent and convenient way.
+ *
+ * A new command can be set up by creating a new `Rsync` instance of
+ * obtaining one through the `build` method.
+ *
+ * @example
+ *   // using the constructor
+ *   var rsync = new Rsync()
+ *       .source('/path/to/source')
+ *       .destination('myserver:destination/');
+ *
+ *   // using the build method with options
+ *   var rsync = Rsync.build({
+ *     source:      '/path/to/source',
+ *     destination: 'myserver:destination/'
+ *   });
+ *
+ * Executing the command can be done using the `execute` method. The command
+ * is executed as a child process and three callbacks can be registered. See
+ * the `execute` method for more details.
+ *
+ * @example
+ *   rsync.execute(function(error, code, cmd) {
+ *     // function called when the child process is finished
+ *   }, function(stdoutChunk) {
+ *     // function called when a chunk of text is received on stdout
+ *   }, function stderrChunk) {
+ *     // function called when a chunk of text is received on stderr
+ *   });
+ *
+ * @author      Mattijs Hoitink <mattijs@monkeyandmachine.com>
+ * @copyright   Copyright (c) 2013, Mattijs Hoitink <mattijs@monkeyandmachine.com>
+ * @license     The MIT License
+ *
+ * @constructor
+ * @param {Object} config Configuration settings for the Rsync wrapper.
+ */
+function Rsync(config) {
+  if (!(this instanceof Rsync)) {
+    return new Rsync(config)
+  }
+
+  // Parse config
+  config = config || {}
+  if (typeof(config) !== 'object') {
+    throw new Error('Rsync config must be an Object')
+  }
+
+  // executable
+  this._executable = hasOP(config, 'executable') ? config.executable : 'rsync'
+
+  // shell
+  this._executableShell = hasOP(config, 'executableShell') ? config.executableShell : '/bin/sh'
+
+  // source(s) and destination
+  this._sources     = []
+  this._destination = ''
+
+  // ordered list of file patterns to include/exclude
+  this._patterns = []
+
+  // list of exlude from getFiles
+  this._excludeFiles = []
+
+  // options
+  this._options = {}
+
+  // output callbacks
+  this._outputHandlers = {
+    stdout: null,
+    stderr: null
+  }
+
+  this._cwd = process.cwd()
+
+  // Allow child_process.spawn env overriding
+  this._env = process.env
+
+  // Debug parameter
+  this._debug = hasOP(config, 'debug') ? config.debug : false
+}
+
+/**
+ * Build a new Rsync command from an options Object.
+ * @param {Object} options
+ * @return {Rsync}
+ */
+Rsync.build = function(options) {
+  var command = new Rsync()
+
+  // Process all options
+  for (var key in options) {
+    if (hasOP(options, key)) {
+      var value = options[key]
+
+      // Only allow calling methods on the Rsync command
+      if (typeof(command[key]) === 'function') {
+        command[key](value)
+      }
+    }
+  }
+
+  return command
+}
+
+/**
+ * Set an option.
+ * @param {String} option
+ * @param mixed value
+ * @return Rsync
+ */
+Rsync.prototype.set = function(option, value) {
+  option = stripLeadingDashes(option)
+  if (option && option.length > 0) {
+    this._options[option] = value || null
+  }
+  return this
+}
+
+/**
+ * Unset an option.
+ * @param {String} option
+ * @return Rsync
+ */
+Rsync.prototype.unset = function(option) {
+  option = stripLeadingDashes(option)
+
+  if (option && Object.keys(this._options).indexOf(option) >= 0) {
+    delete this._options[option]
+  }
+  return this
+}
+
+/**
+ * Set or unset one or more flags. A flag is a single letter option without a value.
+ *
+ * Flags can be presented as a single String, an Array containing Strings or an Object
+ * with the flags as keys.
+ *
+ * When flags are presented as a String or Array the set or unset method will be determined
+ * by the second parameter.
+ * When the flags are presented as an Object the set or unset method will be determined by
+ * the value corresponding to each flag key.
+ *
+ * @param {String|Array|Object} flags
+ * @param {Boolean} set
+ * @return Rsync
+ */
+Rsync.prototype.flags = function(flags, set) {
+  // Do some argument handling
+  if (!arguments.length) {
+    return this
+  }
+  else if (arguments.length === 1) {
+    set = true
+  }
+  else {
+    // There are more than 1 arguments, assume flags are presented as strings
+    flags = Array.prototype.slice.call(arguments)
+
+    // Check if the last argument is a boolean
+    if (typeof(flags[flags.length - 1]) === 'boolean') {
+      set = flags.pop()
+    }
+    else {
+      set = true
+    }
+
+    // Join the remainder of the arguments to treat them as flags
+    flags = flags.join('')
+  }
+
+  // Split multiple flags
+  if (typeof(flags) === 'string') {
+    flags = stripLeadingDashes(flags).split('')
+  }
+
+  // Turn array into an object
+  if (isArray(flags)) {
+    var obj = {}
+    flags.forEach(function(f) {
+      obj[f] = set
+    })
+    flags = obj
+  }
+
+  // set/unset each flag
+  for (var key in flags) {
+    if (hasOP(flags, key)) {
+      var method = (flags[key]) ? 'set' : 'unset'
+      this[method](stripLeadingDashes(key))
+    }
+  }
+
+  return this
+}
+
+/**
+ * Check if an option is set.
+ * @param {String} option
+ * @return {Boolean}
+ */
+Rsync.prototype.isSet = function(option) {
+  option = stripLeadingDashes(option)
+  return Object.keys(this._options).indexOf(option) >= 0
+}
+
+/**
+ * Get an option by name.
+ * @param {String} name
+ * @return mixed
+ */
+Rsync.prototype.option = function(name) {
+  name = stripLeadingDashes(name)
+  return this._options[name]
+}
+
+/**
+ * Register a list of file patterns to include/exclude in the transfer. Patterns can be
+ * registered as an array of Strings or Objects.
+ *
+ * When registering a pattern as a String it must be prefixed with a `+` or `-` sign to
+ * signal include or exclude for the pattern. The sign will be stripped of and the
+ * pattern will be added to the ordered pattern list.
+ *
+ * When registering the pattern as an Object it must contain the `action` and
+ * `pattern` keys where `action` contains the `+` or `-` sign and the `pattern`
+ * key contains the file pattern, without the `+` or `-` sign.
+ *
+ * @example
+ *   // on an existing rsync object
+ *   rsync.patterns(['-docs', { action: '+', pattern: '/subdir/*.py' }]);
+ *
+ *   // using Rsync.build for a new rsync object
+ *   rsync = Rsync.build({
+ *     ...
+ *     patterns: [ '-docs', { action: '+', pattern: '/subdir/*.py' }]
+ *     ...
+ *   })
+ *
+ * @param {Array} patterns
+ * @return Rsync
+ */
+Rsync.prototype.patterns = function(patterns) {
+  if (arguments.length > 1) {
+    patterns = Array.prototype.slice.call(arguments, 0)
+  }
+  if (!isArray(patterns)) {
+    patterns = [ patterns ]
+  }
+
+  patterns.forEach(function(pattern) {
+    var action = '?'
+    if (typeof(pattern) === 'string') {
+      action  = pattern.charAt(0)
+      pattern = pattern.substring(1)
+    }
+    else if (
+      typeof(pattern) === 'object' &&
+            hasOP(pattern, 'action') &&
+            hasOP(pattern, 'pattern')
+    ) {
+      action  = pattern.action
+      pattern = pattern.pattern
+    }
+
+    // Check if the pattern is an include or exclude
+    if (action === '-') {
+      this.exclude(pattern)
+    }
+    else if (action === '+') {
+      this.include(pattern)
+    }
+    else {
+      throw new Error('Invalid pattern: ' + pattern)
+    }
+  }, this)
+
+  return this
+}
+
+/**
+ * Exclude a file pattern from transfer. The pattern will be appended to the ordered list
+ * of patterns for the rsync command.
+ *
+ * @param {String|Array} patterns
+ * @return Rsync
+ */
+Rsync.prototype.exclude = function(patterns) {
+  if (arguments.length > 1) {
+    patterns = Array.prototype.slice.call(arguments, 0)
+  }
+  if (!isArray(patterns)) {
+    patterns = [ patterns ]
+  }
+
+  patterns.forEach(function(pattern) {
+    this._patterns.push({ action:  '-', pattern: pattern })
+  }, this)
+
+  return this
+}
+
+/**
+ * Exclude a file of globs/patterns. The file path will to a list
+ * of file paths to be used with --exclude-from.
+ *
+ * @param {String|Array} patterns
+ * @return Rsync
+ */
+Rsync.prototype.excludeFrom = function(filePaths) {
+  if (arguments.length > 1) {
+    filePaths = Array.prototype.slice.call(arguments, 0)
+  }
+  if (!isArray(filePaths)) {
+    filePaths = [ filePaths ]
+  }
+
+  filePaths.forEach(function(filePath) {
+    this._excludeFiles.push(filePath)
+  }, this)
+
+  return this
+}
+
+/**
+ * Include a file pattern for transfer. The pattern will be appended to the ordered list
+ * of patterns for the rsync command.
+ *
+ * @param {String|Array} patterns
+ * @return Rsync
+ */
+Rsync.prototype.include = function(patterns) {
+  if (arguments.length > 1) {
+    patterns = Array.prototype.slice.call(arguments, 0)
+  }
+  if (!isArray(patterns)) {
+    patterns = [ patterns ]
+  }
+
+  patterns.forEach(function(pattern) {
+    this._patterns.push({ action:  '+', pattern: pattern })
+  }, this)
+
+  return this
+}
+
+/**
+ * Get the command that is going to be executed.
+ * @return {String}
+ */
+Rsync.prototype.command = function() {
+  return this.executable() + ' ' + this.args().join(' ')
+}
+
+/**
+ * String representation of the Rsync command. This is the command that is
+ * going to be executed when calling Rsync::execute.
+ * @return {String}
+ */
+Rsync.prototype.toString = Rsync.prototype.command
+
+/**
+ * Get the arguments for the rsync command.
+ * @return {Array}
+ */
+Rsync.prototype.args = function() {
+  // Gathered arguments
+  var args = []
+
+  // Add options. Short options (one letter) without values are gathered together.
+  // Long options have a value but can also be a single letter.
+  var short = []
+  var long  = []
+
+  // Split long and short options
+  for (var key in this._options) {
+    if (hasOP(this._options, key)) {
+      var value = this._options[key]
+      var noval = (value === null || value === undefined)
+
+      // Check for short option (single letter without value)
+      if (key.length === 1 && noval) {
+        short.push(key)
+      }
+      else {
+        if (isArray(value)) {
+          value.forEach(function (val) {
+            long.push(buildOption(key, val, escapeShellArg))
+          })
+        }
+        else {
+          long.push(buildOption(key, value, escapeShellArg))
+        }
+      }
+
+    }
+  }
+
+  // Add combined short options if any are present
+  if (short.length > 0) {
+    args.push('-' + short.join(''))
+  }
+
+  // Add long options if any are present
+  if (long.length > 0) {
+    args = args.concat(long)
+  }
+
+  // add exclude-from file paths
+  this._excludeFiles.forEach(function(file) {
+    args.push(`--exclude-from=${escapeFileArg(file)}`)
+  })
+
+  // Add includes/excludes in order
+  this._patterns.forEach(function(def) {
+    if (def.action === '-') {
+      args.push(buildOption('exclude', def.pattern, escapeFileArg))
+    }
+    else if (def.action === '+') {
+      args.push(buildOption('include', def.pattern, escapeFileArg))
+    }
+    else {
+      debug(this, 'Unknown pattern action ' + def.action)
+    }
+  })
+
+  // Add sources
+  if (this.source().length > 0) {
+    args = args.concat(this.source().map(escapeFileArg))
+  }
+
+  // Add destination
+  if (this.destination()) {
+    args.push(escapeFileArg(this.destination()))
+  }
+
+  return args
+}
+
+/**
+ * Get and set rsync process cwd directory.
+ *
+ * @param  {string} cwd= Directory path relative to current process directory.
+ * @return {string} Return current _cwd.
+ */
+Rsync.prototype.cwd = function(cwd) {
+  if (arguments.length > 0) {
+    if (typeof cwd !== 'string') {
+      throw new Error('Directory should be a string')
+    }
+
+    this._cwd = path.resolve(cwd)
+  }
+
+  return this._cwd
+}
+
+/**
+ * Get and set rsync process environment variables
+ *
+ * @param  {string} env= Environment variables
+ * @return {string} Return current _env.
+ */
+Rsync.prototype.env = function(env) {
+  if (arguments.length > 0) {
+    if (typeof env !== 'object') {
+      throw new Error('Environment should be an object')
+    }
+
+    this._env = env
+  }
+
+  return this._env
+}
+
+/**
+ * Register an output handlers for the commands stdout and stderr streams.
+ * These functions will be called once data is streamed on one of the output buffers
+ * when the command is executed using `execute`.
+ *
+ * Only one callback function can be registered for each output stream. Previously
+ * registered callbacks will be overridden.
+ *
+ * @param {Function} stdout     Callback Function for stdout data
+ * @param {Function} stderr     Callback Function for stderr data
+ * @return Rsync
+ */
+Rsync.prototype.output = function(stdout, stderr) {
+  // Check for single argument so the method can be used with Rsync.build
+  if (arguments.length === 1 && Array.isArray(stdout)) {
+    stderr = stdout[1]
+    stdout = stdout[0]
+  }
+
+  if (typeof(stdout) === 'function') {
+    this._outputHandlers.stdout = stdout
+  }
+  if (typeof(stderr) === 'function') {
+    this._outputHandlers.stderr = stderr
+  }
+
+  return this
+}
+
+/**
+ * Execute the rsync command.
+ *
+ * The callback function is called with an Error object (or null when there was none),
+ * the exit code from the executed command and the executed command as a String.
+ *
+ * When stdoutHandler and stderrHandler functions are provided they will be used to stream
+ * data from stdout and stderr directly without buffering.
+ *
+ * @param {Function} callback       Called when rsync finishes (optional)
+ * @param {Function} stdoutHandler  Called on each chunk received from stdout (optional)
+ * @param {Function} stderrHandler  Called on each chunk received from stderr (optional)
+ */
+Rsync.prototype.execute = function(callback, stdoutHandler, stderrHandler) {
+  // Register output handlers
+  this.output(stdoutHandler, stderrHandler)
+
+  // Execute the command as a child process
+  // see https://github.com/joyent/node/blob/937e2e351b2450cf1e9c4d8b3e1a4e2a2def58bb/lib/child_process.js#L589
+  var cmdProc
+  if ('win32' === process.platform) {
+    cmdProc = spawn('cmd.exe', ['/s', '/c', '"' + this.command() + '"'],
+      { stdio: 'pipe', windowsVerbatimArguments: true, cwd: this._cwd, env: this._env })
+  }
+  else {
+    cmdProc = spawn(this._executableShell, ['-c', this.command()],
+      { stdio: 'pipe', cwd: this._cwd, env: this._env })
+  }
+
+  // Capture stdout and stderr if there are output handlers configured
+  if (typeof(this._outputHandlers.stdout) === 'function') {
+    cmdProc.stdout.on('data', this._outputHandlers.stdout)
+  }
+  if (typeof(this._outputHandlers.stderr) === 'function') {
+    cmdProc.stderr.on('data', this._outputHandlers.stderr)
+  }
+
+  // Wait for the command to finish
+  cmdProc.on('close', function(code) {
+    var error = null
+
+    // Check rsyncs error code
+    // @see http://bluebones.net/2007/06/rsync-exit-codes/
+    if (code !== 0) {
+      error = new Error('rsync exited with code ' + code)
+    }
+
+    // Check for callback
+    if (typeof(callback) === 'function') {
+      callback(error, code, this.command())
+    }
+  }.bind(this))
+
+  // Return the child process object so it can be cleaned up
+  // if the process exits
+  return(cmdProc)
+}
+
+/**
+ * Get or set the debug property.
+ *
+ * The property is set to the boolean provided so unsetting the debug
+ * property has to be done by passing false to this method.
+ *
+ * @function
+ * @name debug
+ * @memberOf Rsync.prototype
+ * @param {Boolean} debug the value of the debug property (optional)
+ * @return {Rsync|Boolean}
+ */
+createValueAccessor('debug')
+
+/**
+ * Get or set the executable to use for the rsync process.
+ *
+ * When setting the executable path the Rsync instance is returned for
+ * the fluent interface. Otherwise the configured executable path
+ * is returned.
+ *
+ * @function
+ * @name executable
+ * @memberOf Rsync.prototype
+ * @param {String} executable path to the executable (optional)
+ * @return {Rsync|String}
+ */
+createValueAccessor('executable')
+
+/**
+ * Get or set the shell to use on non-Windows (Unix or Mac OS X) systems.
+ *
+ * When setting the shell the Rsync instance is returned for the
+ * fluent interface. Otherwise the configured shell is returned.
+ *
+ * @function
+ * @name executableShell
+ * @memberOf Rsync.prototype
+ * @param {String} shell to use on non-Windows systems (optional)
+ * @return {Rsync|String}
+ */
+createValueAccessor('executableShell')
+
+/**
+ * Get or set the destination for the transfer.
+ *
+ * When setting the destination the Rsync instance is returned for
+ * the fluent interface. Otherwise the configured destination path
+ * is returned.
+ *
+ * @function
+ * @name destination
+ * @memberOf Rsync.prototype
+ * @param {String} destination the destination (optional)
+ * @return {Rsync|String}
+ */
+createValueAccessor('destination')
+
+/**
+ * Add one or more sources for the command or get the list of configured
+ * sources.
+ *
+ * The sources are appended to the list of known sources if they were not
+ * included yet and the Rsync instance is returned for the fluent
+ * interface. Otherwise the configured list of source is returned.
+ *
+ * @function
+ * @name source
+ * @memberOf Rsync.prototype
+ * @param {String|Array} sources the source or list of sources to configure (optional)
+ * @return {Rsync|Array}
+ */
+createListAccessor('source', '_sources')
+
+/**
+ * Set the shell to use when logging in on a remote server.
+ *
+ * This is the same as setting the `rsh` option.
+ *
+ * @function
+ * @name shell
+ * @memberOf Rsync.prototype
+ * @param {String} shell the shell option to use
+ * @return {Rsync}
+ */
+exposeLongOption('rsh', 'shell')
+
+/**
+ * Add a chmod instruction to the command.
+ *
+ * @function
+ * @name chmod
+ * @memberOf Rsync.prototype
+ * @param {String|Array}
+ * @return {Rsync|Array}
+ */
+exposeMultiOption('chmod', 'chmod')
+
+/**
+ * Set the delete flag.
+ *
+ * This is the same as setting the `--delete` commandline flag.
+ *
+ * @function
+ * @name delete
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('delete')
+
+/**
+ * Set the progress flag.
+ *
+ * This is the same as setting the `--progress` commandline flag.
+ *
+ * @function
+ * @name progress
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('progress')
+
+/**
+ * Set the archive flag.
+ *
+ * @function
+ * @name archive
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('a', 'archive')
+
+/**
+ * Set the compress flag.
+ *
+ * @function
+ * @name compress
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('z', 'compress')
+
+/**
+ * Set the recursive flag.
+ *
+ * @function
+ * @name recursive
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('r', 'recursive')
+
+/**
+ * Set the update flag.
+ *
+ * @function
+ * @name update
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('u', 'update')
+
+/**
+ * Set the quiet flag.
+ *
+ * @function
+ * @name quiet
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('q', 'quiet')
+
+/**
+ * Set the dirs flag.
+ *
+ * @function
+ * @name dirs
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('d', 'dirs')
+
+/**
+ * Set the links flag.
+ *
+ * @function
+ * @name links
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('l', 'links')
+
+/**
+ * Set the dry flag.
+ *
+ * @function
+ * @name dry
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('n', 'dry')
+
+/**
+ * Set the hard links flag preserving hard links for the files transmitted.
+ *
+ * @function
+ * @name hardLinks
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('H', 'hardLinks')
+
+/**
+ * Set the perms flag.
+ *
+ * @function
+ * @name perms
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('p', 'perms')
+
+/**
+ * Set the executability flag to preserve executability for the files
+ * transmitted.
+ *
+ * @function
+ * @name executability
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('E', 'executability')
+
+/**
+ * Set the group flag to preserve the group permissions of the files
+ * transmitted.
+ *
+ * @function
+ * @name group
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('g', 'group')
+
+/**
+ * Set the owner flag to preserve the owner of the files transmitted.
+ *
+ * @function
+ * @name owner
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('o', 'owner')
+
+/**
+ * Set the acls flag to preserve the ACLs for the files transmitted.
+ *
+ * @function
+ * @name acls
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('A', 'acls')
+
+/**
+ * Set the xattrs flag to preserve the extended attributes for the files
+ * transmitted.
+ *
+ * @function
+ * @name xattrs
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('X', 'xattrs')
+
+/**
+ * Set the devices flag to preserve device files in the transfer.
+ *
+ * @function
+ * @name devices
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('devices')
+
+/**
+ * Set the specials flag to preserve special files.
+ *
+ * @function
+ * @name specials
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('specials')
+
+/**
+ * Set the times flag to preserve times for the files in the transfer.
+ *
+ * @function
+ * @name times
+ * @memberOf Rsync.prototype
+ * @return {Rsync}
+ */
+exposeShortOption('t', 'times')
+
+// our awesome export product
+module.exports = Rsync
+
+/* **** */
+
+/**
+ * Create a chainable function on the Rsync prototype for getting and setting an
+ * internal value.
+ * @param {String} name
+ * @param {String} internal
+ */
+function createValueAccessor(name, internal) {
+  var container = internal || '_' + name
+
+  Rsync.prototype[name] = function(value) {
+    if (!arguments.length) return this[container]
+    this[container] = value
+    return this
+  }
+}
+
+/**
+ * @param {String} name
+ * @param {String} internal
+ */
+function createListAccessor(name, internal) {
+  var container = internal || '_' + name
+
+  Rsync.prototype[name] = function(value) {
+    if (!arguments.length) return this[container]
+
+    if (isArray(value)) {
+      value.forEach(this[name], this)
+    }
+    else if (typeof(value) !== 'string') {
+      throw new Error('Value for Rsync::' + name + ' must be a String')
+    }
+    else if (this[container].indexOf(value) < 0) {
+      this[container].push(value)
+    }
+
+    return this
+  }
+}
+
+/**
+ * Create a shorthand method on the Rsync prototype for setting and unsetting a simple option.
+ * @param {String} option
+ * @param {String} name
+ */
+function exposeShortOption(option, name) {
+  name = name || option
+
+  Rsync.prototype[name] = function(set) {
+    // When no arguments are passed in assume the option
+    // needs to be set
+    if (!arguments.length) set = true
+
+    var method = (set) ? 'set' : 'unset'
+    return this[method](option)
+  }
+}
+
+/**
+ * Create a function for an option that can be set multiple time. The option
+ * will accumulate all values.
+ *
+ * @param {String} option
+ * @param {[String]} name
+ */
+function exposeMultiOption(option, name) {
+  name = name || option
+
+  Rsync.prototype[name] = function(value) {
+    // When not arguments are passed in assume the options
+    // current value is requested
+    if (!arguments.length) return this.option(option)
+
+    if (!value) {
+      // Unset the option on falsy
+      this.unset(option)
+    }
+    else if (isArray(value)) {
+      // Call this method for each array value
+      value.forEach(this[name], this)
+    }
+    else {
+      // Add the value
+      var current = this.option(option)
+      if (!current) {
+        value = [ value ]
+      }
+      else if (!isArray(current)) {
+        value = [ current, value ]
+      }
+      else {
+        value = current.concat(value)
+      }
+
+      this.set(option, value)
+    }
+
+    return this
+  }
+}
+
+/**
+ * Expose an rsync long option on the Rsync prototype.
+ * @param {String} option   The option to expose
+ * @param {String} name     An optional alternative name for the option.
+ */
+function exposeLongOption(option, name) {
+  name = name || option
+
+  Rsync.prototype[name] = function(value) {
+    // When not arguments are passed in assume the options
+    // current value is requested
+    if (!arguments.length) return this.option(option)
+
+    var method = (value) ? 'set' : 'unset'
+    return this[method](option, value)
+  }
+}
+
+/**
+ * Build an option for use in a shell command.
+ *
+ * @param {String} name
+ * @param {String} value
+ * @param {Function|boolean} escapeArg
+ * @return {String}
+ */
+function buildOption(name, value, escapeArg) {
+  if (typeof escapeArg === 'boolean') {
+    escapeArg = (!escapeArg) ? noop : null
+  }
+
+  if (typeof escapeArg !== 'function') {
+    escapeArg = escapeShellArg
+  }
+
+  // Detect single option key
+  var single = (name.length === 1) ? true : false
+
+  // Decide on prefix and value glue
+  var prefix = (single) ? '-' : '--'
+  var glue   = (single) ? ' ' : '='
+
+  // Build the option
+  var option = prefix + name
+  if (arguments.length > 1 && value) {
+    value   = escapeArg(String(value))
+    option += glue + value
+  }
+
+  return option
+}
+
+/**
+ * Escape an argument for use in a shell command when necessary.
+ * @param {String} arg
+ * @return {String}
+ */
+function escapeShellArg(arg) {
+  if (!/(["'`\\$ ])/.test(arg)) {
+    return arg
+  }
+  return '"' + arg.replace(/(["'`\\$])/g, '\\$1') + '"'
+}
+
+/**
+ * Escape a filename for use in a shell command.
+ * @param {String} filename the filename to escape
+ * @return {String} the escaped version of the filename
+ */
+function escapeFileArg(filename) {
+  filename = filename.replace(/(["'`\s\\\(\)\\$])/g,'\\$1')
+  if (!/(\\\\)/.test(filename)) {
+    return filename
+  }
+  // Under Windows rsync (with cygwin) and OpenSSH for Windows
+  // (http://www.mls-software.com/opensshd.html) are using
+  // standard linux directory separator so need to replace it
+  if ('win32' === process.platform) {
+    filename = filename.replace(/\\\\/g,'/').replace(/^["]?[A-Z]\:\//ig,'/')
+  }
+  return filename
+}
+
+/**
+ * Strip the leading dashes from a value.
+ * @param {String} value
+ * @return {String}
+ */
+function stripLeadingDashes(value) {
+  if (typeof(value) === 'string') {
+    value = value.replace(/^[\-]*/, '')
+  }
+
+  return value
+}
+
+/**
+ * Simple function for checking if a value is an Array. Will use the native
+ * Array.isArray method if available.
+ * @private
+ * @param {Mixed} value
+ * @return {Boolean}
+ */
+function isArray(value) {
+  if (typeof(Array.isArray) === 'function') {
+    return Array.isArray(value)
+  }
+  else {
+    return toString.call(value) == '[object Array]'
+  }
+}
+
+/**
+ * Simple hasOwnProperty wrapper. This will call hasOwnProperty on the obj
+ * through the Object prototype.
+ * @private
+ * @param {Object} obj  The object to check the property on
+ * @param {String} key  The name of the property to check
+ * @return {Boolean}
+ */
+function hasOP(obj, key) {
+  return Object.prototype.hasOwnProperty.call(obj, key)
+}
+
+function noop() {}
+
+/**
+ * Simple debug printer.
+ *
+ * @private
+ * @param {Rsync} cmd
+ * @param {String} message
+ */
+function debug(cmd, message) {
+  if (!cmd._debug) return
+}
diff --git a/src/sync.js b/src/sync.js
new file mode 100644
index 0000000..7811d3b
--- /dev/null
+++ b/src/sync.js
@@ -0,0 +1,195 @@
+// local imports
+import Rsync from './rsync'
+// native imports
+import { EventEmitter as Emitter } from 'events'
+import { dirname } from 'path'
+// third party elements
+import merge from 'aggregation/es6'
+import { readFile } from 'fs-read-data'
+import Conf from 'conf'
+import getFiles from 'globby'
+import pathExists from 'path-exists'
+import to from 'await-to-js'
+// uci imports
+import logger from '@uci/logger'
+let log = {} // declare module wide log to be set during construction
+
+class Sync extends merge(Rsync, Emitter) {
+  constructor(opts = {}) {
+    super()
+    log = logger({ package:'@uci/sync'})
+    this.opts = opts
+    this.cli = opts.cli
+    this.config = new Conf({projectName:'sync'})
+    this.jobsDir = process.env.SYNC_JOBS_DIR  || opts.jobsDir || this.config.get('jobsDir') || dirname(this.config.path)
+    this.sshDir = opts.sshDir || this.config.get('sshDir') || `${this.jobsDir}/ssh`
+    this.optionsDir = opts.optionsDir || this.config.get('optionsDir') || `${this.jobsDir}/options`
+    log.debug({configsDir:this.configsDir, msg:'configuration files directory'})
+    this.jobs = []
+  }
+
+  setConfig(name,val) { this.config.set(name,val)}
+  getConfig(name) { return this.config.get(name)}
+
+  async setJobsDir(dir='',save) {
+    let res = await pathExists(dir)
+    if(res){
+      this.jobsDir=dir
+      if(save) this.setDefaultJobsDir()
+      return res
+    }
+    else {
+      log.warn(`${dir} path does not exist  - Jobs Directory remains ${this.jobsDir}`)
+      return res
+    }
+  }
+
+  setDefaultJobsDir() { this.config.set('jobsDir', this.jobsDir)}
+  getDefaultJobsDir() { this.config.get('jobsDir', this.jobsDir) }
+
+  async listJobFiles(dir) {
+    return await getFiles(['**','*.yaml','*.yml'],{ onlyfiles:true, cwd:dir || this.jobsDir})
+  }
+
+  async readOptionsFile(filePath,type='job') {
+    let dir = {job:this.jobsDir,options:this.optionsDir,ssh:this.sshDir}
+    let [err,res] = await to(readFile(`${dir[type]}/${filePath}`))
+    if (err) {
+      [err,res] = await to(readFile(filePath))
+      if (err) {
+        err = {filePath:filePath, error:err, type:type, dir:dir[type], msg:`unable to read ${filePath} options file`}
+        log.warn(err)
+        return err
+      }
+    }
+    return res
+  }
+
+  live() {
+    this.unset('n')
+  }
+
+  sshu (file) {
+    if (file && (typeof options==='string')) {
+      this.shell(`"sshu -c ${file} "`)
+    }
+  }
+
+  async ssh (options) {
+    if (!this.destination()) {
+      log.warn({ssh:options, msg:'aborting ssh options - destination must be set before processing ssh option'})
+      return this
+    }
+    if (typeof options==='string') {  // options is a filepath for ssh options
+      let options = await this.readOptionsFile(options,'ssh')
+      if (options.error) return this
+    }
+
+    if (options.host)
+    {
+      let username = options.username ? options.username+'@' : ''
+      this.destination(`${username}${options.host}:${this.destination()}`)
+    }
+    else {
+      log.warn({ssh:options, msg:'aborting ssh options, missing host value is required '})
+      return this
+    }
+    let cmd = []
+    // TODO allow all openssh client options
+    if (options.privateKeyPath)  cmd.push(`-i ${options.privateKeyPath}`)
+    if (options.port)  cmd.push(`-p ${options.port}`)
+    if (options.configFile)  cmd.push(`-F ${options.privateKeyPath}`)
+    if (cmd) this.shell(cmd.join(' '))
+
+    return this
+  }
+
+  set (option={},value) {
+    if (typeof option === 'string') super.set(option,value)  // pass through
+    else {
+      // if (!Array.isArray(option)) { option = [option] }
+      option.forEach( opt => {
+        console.log('---',option,opt)
+        typeof opt==='string' ? super.set(opt) : super.set(...Object.entries(opt).flat())
+      })
+    }
+    return this
+  }
+
+
+  async loadJob (options) {
+    if (typeof options ==='string') options = await this.readOptionsFile(options,'job')
+    if (!options.error) {
+      for (const option in options) {
+        if(option === 'optionsFile') {
+          let opts = await this.readOptionsFile(options.optionsFile,'options')
+          if (!opts.error) {
+            Object.keys(opts).forEach( opt => {
+              this.processOption(opt,opts[opt])
+            })
+          }
+        }
+        else {
+          this.processOption(option,options[option])
+        }
+      } // end loop
+      this.dry()  // dry run by default must .live() .unset('n')
+    }
+  }
+
+  // executes a method on the instance (might be in prototype chain) which may take a value(s)
+  processOption (method, value)  {
+    if (typeof(this[method]) === 'function') {
+      this[method](value)
+    }
+
+  }
+
+  queueJobs () {}
+  runJobs () {}
+
+  async execute(opts={}) {
+    log.info({cmd:this.command(), msg:'running rsync command'})
+    const superexecute = super.execute.bind(this)
+    return new Promise((resolve, reject) => {
+      let status
+      let errors
+      this.rsyncpid = superexecute(
+        function(err, code, cmd) {
+          if (err) {
+            log.fatal({error:err, code:code, cmd:cmd, msg:'error during sync'})
+            reject ({error:err, code:code, cmd:cmd, msg:'error during sync'})
+          }
+          if (errors) log.warn({errors: errors, cmd:cmd, msg:'sync ran but with with errors'})
+          log.info({cmd:cmd, status:status, msg:'sync run'})
+          resolve({cmd:cmd, errors:errors, status:status, msg:'sync run'})
+        }, function(data) {
+          status += data.toString()
+          if (opts.cli) process.stdout.write(data.toString())
+        },
+        function(data) {
+          errors += data.toString()
+          if (opts.cli) process.stderr.write(data.toString())
+        }
+      )
+    })
+  }
+
+}// end Class Sync
+
+export default Sync
+
+
+
+
+// function isPlainObject (obj) {
+//   return Object.prototype.toString.call(obj) === '[object Object]'
+// }
+//
+// function escapeSpaces (str) {
+//   if (typeof str === 'string') {
+//     return str.replace(/\b\s/g, '\\ ')
+//   } else {
+//     return path
+//   }
+// }
diff --git a/src/watcher.js b/src/watcher.js
new file mode 100644
index 0000000..f7a3ad8
--- /dev/null
+++ b/src/watcher.js
@@ -0,0 +1,41 @@
+import { EventEmitter as Emitter } from 'events'
+import path from 'path'
+import chokidar from 'chokidar'
+
+class Watcher extends Emitter {
+  constructor(options) {
+    super()
+    // pass in ignores
+    const opts = {
+      ignored: '**/node_modules/**',
+      ignoreInitial: true
+    }
+    const watcher = chokidar.watch(options.source, opts)
+    this.watcher = watcher
+  }
+
+  start() {
+    const handler = (type, f) => {
+      const fname = path.basename(f)
+      if ( fname.toLowerCase() === 'package.json')
+        if (type !=='change') {
+          this.emit('error',new Error('package.json was added or removed, ignoring sync and reinstall'))
+          return
+        } else{
+          this.emit('install', f)
+        }
+      this.emit('sync', f)
+    } // end handler
+    this.watcher
+      .on('add', handler.bind(this, 'add'))
+      .on('change', handler.bind(this, 'change'))
+      .on('unlink', handler.bind(this, 'remove'))
+  }
+
+  stop() {
+    this.watcher.close()
+  }
+
+}
+
+export default Watcher
diff --git a/test/jobs/local.yaml b/test/jobs/local.yaml
new file mode 100644
index 0000000..ce6c5bf
--- /dev/null
+++ b/test/jobs/local.yaml
@@ -0,0 +1,8 @@
+source: ./test/repo
+destination: ./test/dest
+flags: 'av'
+excludeFrom:
+      - ./test/repo/.gitignore
+set:
+  - delete
+  - delete-excluded
diff --git a/test/jobs/options/mirror.yaml b/test/jobs/options/mirror.yaml
new file mode 100644
index 0000000..6de96ac
--- /dev/null
+++ b/test/jobs/options/mirror.yaml
@@ -0,0 +1,20 @@
+flags: [a,H]
+# set:
+#     - exclude-from: '.gitignore'
+#     - exclude-from: '.npmignore'
+# exclude:
+#     - 'test1'
+#     - 'test2'
+#     - 'test3'
+# set:
+#   exclude-from: '.gitignore'
+excludeFrom:
+      - ./test/repo/.gitignore
+      - ./test/repo/.npmignore
+
+# ssh:
+#   host: giskard.kebler.net
+#   port: 33
+#   username: sysadmin
+#   keyFilePath: '~/.ssh/keyfile'
+#   ssh: 'ssh'
diff --git a/test/jobs/sample.yaml b/test/jobs/sample.yaml
new file mode 100644
index 0000000..b93e8d2
--- /dev/null
+++ b/test/jobs/sample.yaml
@@ -0,0 +1,12 @@
+source: ./test/repo
+destination: /opt/
+# optionsFile: 'mirror'
+flags: 'av'
+ssh:
+  host: switches.kebler.net
+  username: sysadmin
+excludeFrom:
+      - ./test/repo/.gitignore
+set:
+  - delete
+  - delete-excluded
diff --git a/test/jobs/ssh/switches.yaml b/test/jobs/ssh/switches.yaml
new file mode 100644
index 0000000..9ab282b
--- /dev/null
+++ b/test/jobs/ssh/switches.yaml
@@ -0,0 +1,7 @@
+host: switches.kebler.net
+username: sysadmin
+# agent: /run/user/1000/ssh-agent.socket
+# agentenv: SSH_AUTH_SOCK
+privateKeyPath: /home/david/.ssh/privatekeys/sysadmin.kebler.net
+port: 22
+#passphrase: '51535560'
diff --git a/test/repo/.gitignore b/test/repo/.gitignore
new file mode 100644
index 0000000..25fbf5a
--- /dev/null
+++ b/test/repo/.gitignore
@@ -0,0 +1,2 @@
+node_modules/
+coverage/
diff --git a/test/repo/.npmignore b/test/repo/.npmignore
new file mode 100644
index 0000000..02078d5
--- /dev/null
+++ b/test/repo/.npmignore
@@ -0,0 +1,5 @@
+tests/
+test/
+*.test.js
+testing/
+example/
diff --git a/test/sync.test.js b/test/sync.test.js
new file mode 100644
index 0000000..d61ff3d
--- /dev/null
+++ b/test/sync.test.js
@@ -0,0 +1,58 @@
+import Sync from '../src/sync'
+import to from 'await-to-js'
+import { expect } from 'chai'
+import { it } from 'mocha'
+import logger from '@uci/logger'
+// pause = require('@uci/utils').pPause
+
+describe('Sync Class Testing ',async ()=> {
+
+  let log = logger({})
+  before(async () => {
+    // log = logger({ package:'@uci/sync', id: 'sync-test' })
+    let sync = new Sync()
+    // await sync.loadJob('local')
+    // console.log('command to be run',sync.command())
+    // console.log('===',sync.cwd())
+    // // sync.execute({cli:true})
+    // sync.live()
+    // sync.execute({cli:true})
+    await sync.loadJob('sample')
+    console.log('command to be run',sync.command())
+    sync.live()
+    sync.execute({cli:true})
+
+    // await sync.configure('./test/config/sync')
+    // log.info({cmd:sync.command(), msg:'Rsync Command that will Run'})
+    // log.info(`making connection to ${remote.opts.host}`)
+    // log.info('ready for testing')
+  })
+
+  // after(async () => {
+  //   remote.close()
+  // })
+
+  it('can sync a directory' , async function () {
+    // let [err,res] = await to(remote.exec('cd /opt && pwd'))
+    // if (err) {
+    //   log.info('error running command aborting test', err)
+    //   return
+    // }
+    // log.info(`result of remote command ${res.command} => ${res.reply.toString().trim()}`)
+    expect('test', 'test failed').to.equal('/opt')
+  })
+
+})
+
+// function hooks(remote) {
+//
+
+// // beforeEach(async() => {
+// // await someasyncfunctiontodobeforeeachtest()
+// // })
+//
+// // after(async() => {
+// // await someasyncfunctiontodoaftereeachtest()
+// // })
+//
+// }