working sync module extending the node-rsync module

.eslintrc.js

@@ -0,0 +1,37 @@
+module.exports = {
+  "ecmaFeatures": {
+    "modules": true,
+    "spread" : true,
+    "restParams" : true
+  },
+  // "plugins": [
+  // "unicorn"
+  //  ],
+  "env": {
+    "es6": true,
+    "node": true,
+    "mocha": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 2017,
+    "sourceType": "module"
+  },
+  "extends": "eslint:recommended",
+  "rules": {
+    "indent": [
+      "error",
+      2
+    ],
+    // "unicorn/no-array-instanceof": "error",
+    "no-console": 0,
+    "semi": ["error", "never"],
+    "linebreak-style": [
+      "error",
+      "unix"
+    ],
+    "quotes": [
+      "error",
+      "single"
+    ]
+  }
+}

.gitignore

@@ -0,0 +1,2 @@
+/node_modules/
+/coverage/

.npmignore

@@ -0,0 +1,6 @@
+tests/
+test/
+*.test.js
+testing/
+example/
+nodemon.json

nodemon.json

@@ -0,0 +1,3 @@
+{
+  "ext": "js, yaml"
+}

package.json

@@ -0,0 +1,51 @@
+{
+  "name": "@uci/remote-code",
+  "version": "0.0.2",
+  "description": "module to copy, maintain, and launch hardware modules on other machines",
+  "main": "src/index.js",
+  "bin": {
+    "ssh": "./bin/ssh.js"
+  },
+  "scripts": {
+    "sync": "node -r esm ./bin/ssh",
+    "testd": "UCI_ENV=dev ./node_modules/.bin/nodemon --exec './node_modules/.bin/mocha -r esm --timeout 30000'",
+    "test": "UCI_ENV=pro UCI_LOG_PATH=./test/test.log ./node_modules/.bin/mocha -r esm --timeout 30000 || exit 0",
+    "testibc": "istanbul cover ./node_modules/.bin/_mocha test/ --report lcovonly -- -R spec --recursive && codecov || true"
+  },
+  "author": "David Kebler",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/uCOMmandIt/uci-remote-code.git"
+  },
+  "keywords": [
+    "node.js",
+    "I2C",
+    "raspberryPi"
+  ],
+  "bugs": {
+    "url": "https://github.com/uCOMmandIt/uci-remote-code/issues"
+  },
+  "homepage": "https://github.com/uCOMmandIt/uci-remote-code#readme",
+  "dependencies": {
+    "@uci/logger": "0.0.10",
+    "aggregation": "^1.2.5",
+    "await-to-js": "^2.1.1",
+    "chokidar": "^2.0.4",
+    "conf": "^2.2.0",
+    "esm": "^3.1.4",
+    "fs-read-data": "^1.0.4",
+    "globby": "^9.0.0",
+    "p-settle": "^2.1.0",
+    "path-exists": "^3.0.0",
+    "yargs": "^12.0.5"
+  },
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "chai-arrays": "^2.0.0",
+    "codecov": "^3.1.0",
+    "istanbul": "^0.4.5",
+    "mocha": "^5.x",
+    "nodemon": "^1.18.9"
+  }
+}

src/sync.js

@@ -0,0 +1,195 @@
+// local imports
+import Rsync from './rsync'
+// native imports
+import { EventEmitter as Emitter } from 'events'
+import { dirname } from 'path'
+// third party elements
+import merge from 'aggregation/es6'
+import { readFile } from 'fs-read-data'
+import Conf from 'conf'
+import getFiles from 'globby'
+import pathExists from 'path-exists'
+import to from 'await-to-js'
+// uci imports
+import logger from '@uci/logger'
+let log = {} // declare module wide log to be set during construction
+
+class Sync extends merge(Rsync, Emitter) {
+  constructor(opts = {}) {
+    super()
+    log = logger({ package:'@uci/sync'})
+    this.opts = opts
+    this.cli = opts.cli
+    this.config = new Conf({projectName:'sync'})
+    this.jobsDir = process.env.SYNC_JOBS_DIR  || opts.jobsDir || this.config.get('jobsDir') || dirname(this.config.path)
+    this.sshDir = opts.sshDir || this.config.get('sshDir') || `${this.jobsDir}/ssh`
+    this.optionsDir = opts.optionsDir || this.config.get('optionsDir') || `${this.jobsDir}/options`
+    log.debug({configsDir:this.configsDir, msg:'configuration files directory'})
+    this.jobs = []
+  }
+
+  setConfig(name,val) { this.config.set(name,val)}
+  getConfig(name) { return this.config.get(name)}
+
+  async setJobsDir(dir='',save) {
+    let res = await pathExists(dir)
+    if(res){
+      this.jobsDir=dir
+      if(save) this.setDefaultJobsDir()
+      return res
+    }
+    else {
+      log.warn(`${dir} path does not exist  - Jobs Directory remains ${this.jobsDir}`)
+      return res
+    }
+  }
+
+  setDefaultJobsDir() { this.config.set('jobsDir', this.jobsDir)}
+  getDefaultJobsDir() { this.config.get('jobsDir', this.jobsDir) }
+
+  async listJobFiles(dir) {
+    return await getFiles(['**','*.yaml','*.yml'],{ onlyfiles:true, cwd:dir || this.jobsDir})
+  }
+
+  async readOptionsFile(filePath,type='job') {
+    let dir = {job:this.jobsDir,options:this.optionsDir,ssh:this.sshDir}
+    let [err,res] = await to(readFile(`${dir[type]}/${filePath}`))
+    if (err) {
+      [err,res] = await to(readFile(filePath))
+      if (err) {
+        err = {filePath:filePath, error:err, type:type, dir:dir[type], msg:`unable to read ${filePath} options file`}
+        log.warn(err)
+        return err
+      }
+    }
+    return res
+  }
+
+  live() {
+    this.unset('n')
+  }
+
+  sshu (file) {
+    if (file && (typeof options==='string')) {
+      this.shell(`"sshu -c ${file} "`)
+    }
+  }
+
+  async ssh (options) {
+    if (!this.destination()) {
+      log.warn({ssh:options, msg:'aborting ssh options - destination must be set before processing ssh option'})
+      return this
+    }
+    if (typeof options==='string') {  // options is a filepath for ssh options
+      let options = await this.readOptionsFile(options,'ssh')
+      if (options.error) return this
+    }
+
+    if (options.host)
+    {
+      let username = options.username ? options.username+'@' : ''
+      this.destination(`${username}${options.host}:${this.destination()}`)
+    }
+    else {
+      log.warn({ssh:options, msg:'aborting ssh options, missing host value is required '})
+      return this
+    }
+    let cmd = []
+    // TODO allow all openssh client options
+    if (options.privateKeyPath)  cmd.push(`-i ${options.privateKeyPath}`)
+    if (options.port)  cmd.push(`-p ${options.port}`)
+    if (options.configFile)  cmd.push(`-F ${options.privateKeyPath}`)
+    if (cmd) this.shell(cmd.join(' '))
+
+    return this
+  }
+
+  set (option={},value) {
+    if (typeof option === 'string') super.set(option,value)  // pass through
+    else {
+      // if (!Array.isArray(option)) { option = [option] }
+      option.forEach( opt => {
+        console.log('---',option,opt)
+        typeof opt==='string' ? super.set(opt) : super.set(...Object.entries(opt).flat())
+      })
+    }
+    return this
+  }
+
+
+  async loadJob (options) {
+    if (typeof options ==='string') options = await this.readOptionsFile(options,'job')
+    if (!options.error) {
+      for (const option in options) {
+        if(option === 'optionsFile') {
+          let opts = await this.readOptionsFile(options.optionsFile,'options')
+          if (!opts.error) {
+            Object.keys(opts).forEach( opt => {
+              this.processOption(opt,opts[opt])
+            })
+          }
+        }
+        else {
+          this.processOption(option,options[option])
+        }
+      } // end loop
+      this.dry()  // dry run by default must .live() .unset('n')
+    }
+  }
+
+  // executes a method on the instance (might be in prototype chain) which may take a value(s)
+  processOption (method, value)  {
+    if (typeof(this[method]) === 'function') {
+      this[method](value)
+    }
+
+  }
+
+  queueJobs () {}
+  runJobs () {}
+
+  async execute(opts={}) {
+    log.info({cmd:this.command(), msg:'running rsync command'})
+    const superexecute = super.execute.bind(this)
+    return new Promise((resolve, reject) => {
+      let status
+      let errors
+      this.rsyncpid = superexecute(
+        function(err, code, cmd) {
+          if (err) {
+            log.fatal({error:err, code:code, cmd:cmd, msg:'error during sync'})
+            reject ({error:err, code:code, cmd:cmd, msg:'error during sync'})
+          }
+          if (errors) log.warn({errors: errors, cmd:cmd, msg:'sync ran but with with errors'})
+          log.info({cmd:cmd, status:status, msg:'sync run'})
+          resolve({cmd:cmd, errors:errors, status:status, msg:'sync run'})
+        }, function(data) {
+          status += data.toString()
+          if (opts.cli) process.stdout.write(data.toString())
+        },
+        function(data) {
+          errors += data.toString()
+          if (opts.cli) process.stderr.write(data.toString())
+        }
+      )
+    })
+  }
+
+}// end Class Sync
+
+export default Sync
+
+
+
+
+// function isPlainObject (obj) {
+//   return Object.prototype.toString.call(obj) === '[object Object]'
+// }
+//
+// function escapeSpaces (str) {
+//   if (typeof str === 'string') {
+//     return str.replace(/\b\s/g, '\\ ')
+//   } else {
+//     return path
+//   }
+// }

src/watcher.js

@@ -0,0 +1,41 @@
+import { EventEmitter as Emitter } from 'events'
+import path from 'path'
+import chokidar from 'chokidar'
+
+class Watcher extends Emitter {
+  constructor(options) {
+    super()
+    // pass in ignores
+    const opts = {
+      ignored: '**/node_modules/**',
+      ignoreInitial: true
+    }
+    const watcher = chokidar.watch(options.source, opts)
+    this.watcher = watcher
+  }
+
+  start() {
+    const handler = (type, f) => {
+      const fname = path.basename(f)
+      if ( fname.toLowerCase() === 'package.json')
+        if (type !=='change') {
+          this.emit('error',new Error('package.json was added or removed, ignoring sync and reinstall'))
+          return
+        } else{
+          this.emit('install', f)
+        }
+      this.emit('sync', f)
+    } // end handler
+    this.watcher
+      .on('add', handler.bind(this, 'add'))
+      .on('change', handler.bind(this, 'change'))
+      .on('unlink', handler.bind(this, 'remove'))
+  }
+
+  stop() {
+    this.watcher.close()
+  }
+
+}
+
+export default Watcher

test/jobs/local.yaml

@@ -0,0 +1,8 @@
+source: ./test/repo
+destination: ./test/dest
+flags: 'av'
+excludeFrom:
+      - ./test/repo/.gitignore
+set:
+  - delete
+  - delete-excluded

test/jobs/options/mirror.yaml

@@ -0,0 +1,20 @@
+flags: [a,H]
+# set:
+#     - exclude-from: '.gitignore'
+#     - exclude-from: '.npmignore'
+# exclude:
+#     - 'test1'
+#     - 'test2'
+#     - 'test3'
+# set:
+#   exclude-from: '.gitignore'
+excludeFrom:
+      - ./test/repo/.gitignore
+      - ./test/repo/.npmignore
+
+# ssh:
+#   host: giskard.kebler.net
+#   port: 33
+#   username: sysadmin
+#   keyFilePath: '~/.ssh/keyfile'
+#   ssh: 'ssh'

test/jobs/sample.yaml

@@ -0,0 +1,12 @@
+source: ./test/repo
+destination: /opt/
+# optionsFile: 'mirror'
+flags: 'av'
+ssh:
+  host: switches.kebler.net
+  username: sysadmin
+excludeFrom:
+      - ./test/repo/.gitignore
+set:
+  - delete
+  - delete-excluded

test/jobs/ssh/switches.yaml

@@ -0,0 +1,7 @@
+host: switches.kebler.net
+username: sysadmin
+# agent: /run/user/1000/ssh-agent.socket
+# agentenv: SSH_AUTH_SOCK
+privateKeyPath: /home/david/.ssh/privatekeys/sysadmin.kebler.net
+port: 22
+#passphrase: '51535560'

test/repo/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+coverage/

test/repo/.npmignore

@@ -0,0 +1,5 @@
+tests/
+test/
+*.test.js
+testing/
+example/

test/sync.test.js

@@ -0,0 +1,58 @@
+import Sync from '../src/sync'
+import to from 'await-to-js'
+import { expect } from 'chai'
+import { it } from 'mocha'
+import logger from '@uci/logger'
+// pause = require('@uci/utils').pPause
+
+describe('Sync Class Testing ',async ()=> {
+
+  let log = logger({})
+  before(async () => {
+    // log = logger({ package:'@uci/sync', id: 'sync-test' })
+    let sync = new Sync()
+    // await sync.loadJob('local')
+    // console.log('command to be run',sync.command())
+    // console.log('===',sync.cwd())
+    // // sync.execute({cli:true})
+    // sync.live()
+    // sync.execute({cli:true})
+    await sync.loadJob('sample')
+    console.log('command to be run',sync.command())
+    sync.live()
+    sync.execute({cli:true})
+
+    // await sync.configure('./test/config/sync')
+    // log.info({cmd:sync.command(), msg:'Rsync Command that will Run'})
+    // log.info(`making connection to ${remote.opts.host}`)
+    // log.info('ready for testing')
+  })
+
+  // after(async () => {
+  //   remote.close()
+  // })
+
+  it('can sync a directory' , async function () {
+    // let [err,res] = await to(remote.exec('cd /opt && pwd'))
+    // if (err) {
+    //   log.info('error running command aborting test', err)
+    //   return
+    // }
+    // log.info(`result of remote command ${res.command} => ${res.reply.toString().trim()}`)
+    expect('test', 'test failed').to.equal('/opt')
+  })
+
+})
+
+// function hooks(remote) {
+//
+
+// // beforeEach(async() => {
+// // await someasyncfunctiontodobeforeeachtest()
+// // })
+//
+// // after(async() => {
+// // await someasyncfunctiontodoaftereeachtest()
+// // })
+//
+// }