refactor of sync and ssh, before splitting off ssh into it's own repo/package.

This commit is contained in:
David Kebler 2019-02-02 18:02:04 -08:00
parent 91c3c305de
commit 9caf1c1da4
19 changed files with 432 additions and 190 deletions

3
bin/ssh-cli.js Normal file
View file

@ -0,0 +1,3 @@
// Set options as a parameter, environment variable, or rc file.
require = require('esm')(module)
module.exports = require('./ssh.js')

41
bin/ssh.js Normal file
View file

@ -0,0 +1,41 @@
#!/usr/bin/env node
import Ssh from '../src/ssh'
import to from 'await-to-js'
import logger from '@uci/logger'
import yargs from 'yargs'
console.log(process.argv)
let arg = yargs
.alias('c','config')
.argv
;
(async () => {
const log = logger({ repo: 'remote-code', bin:'./bin/ssh.js', class:'Ssh', file:'src/ssh.js', id: 'binary' })
const ssh = new Ssh()
console.log('config file', arg)
await ssh.configure(arg.c ? arg.c : null) // default config
log.info(`making connection to ${ssh.opts.host}`)
let [err] = await to(ssh.connect())
if (err) {
log.info({err:err, msg:'unable to connect'})
return err
}
// let cmd = arg._.join(' ')
// console.log(cmd)
// let [err2,res] = await to(ssh.exec(cmd))
// if (err2) {
// log.info({err:err2, command:cmd, msg:'error running command aborting'})
// return
// }
// ssh.close()
// return res
let res = await ssh.shell({cli:true})
console.log('ALL DONE', res.command, res.cmds, )
})().catch(err => {
console.error('FATAL: Failed to Connecct to Remote!\n',err)
})

2
bin/ssh.sh Executable file
View file

@ -0,0 +1,2 @@
DEV_BIN=/mnt/238/nas/david/hacking/active-dev-repos/uci/lib/uci-remote-code/bin
UCI_DEV=true /usr/bin/node $DEV_BIN/ssh-cli $@

3
nodemon.json Normal file
View file

@ -0,0 +1,3 @@
{
"ext": "js, yaml"
}

View file

@ -3,8 +3,12 @@
"version": "0.0.2",
"description": "module to copy, maintain, and launch hardware modules on other machines",
"main": "src/index.js",
"bin": {
"ssh": "./bin/ssh.js"
},
"scripts": {
"testd": "./node_modules/.bin/mocha -r esm --watch --timeout 30000 || exit 0",
"ssh": "node -r esm ./bin/ssh",
"testd": "UCI_DEV=true ./node_modules/.bin/nodemon --exec './node_modules/.bin/mocha -r esm --timeout 30000'",
"test": "UCI_PRO=./test/test.log ./node_modules/.bin/mocha -r esm --timeout 30000 || exit 0",
"testibc": "istanbul cover ./node_modules/.bin/_mocha test/ --report lcovonly -- -R spec --recursive && codecov || true"
},
@ -27,16 +31,19 @@
"@uci/logger": "0.0.9",
"await-to-js": "^2.1.1",
"chokidar": "^2.0.4",
"conf": "^2.2.0",
"fs-read-data": "^1.0.4",
"p-settle": "^2.1.0",
"rsyncwrapper": "^3.0.1",
"ssh2": "^0.8.2"
"ssh2": "^0.8.2",
"yargs": "^12.0.5"
},
"devDependencies": {
"chai": "^4.2.0",
"chai-arrays": "^2.0.0",
"codecov": "^3.1.0",
"esm": "^3.1.4",
"istanbul": "^0.4.5",
"mocha": "^5.x",
"read-yaml-file": "^1.1.0"
"nodemon": "^1.18.9"
}
}

View file

@ -1,39 +0,0 @@
import settle from 'p-settle'
import { promisify } from 'util'
import { readFile } from 'fs'
import path from 'path'
const read = promisify(readFile)
// A helper function to return a list of paths to ignore from .npmignore, .gitignore, .rcignore
function ignore (repoPath,files) {
// console.log('additional files', files)
let ignoreList = []
let ignoreFiles = ['.npmignore','.gitignore','.rcignore']
if (Array.isArray(files)) {
ignoreFiles=[...ignoreFiles,...files]
} else {
if (files) ignoreFiles.push(files)
}
// each set in an the array is new line delimited set of ignore patterns
return settle(ignoreFiles.map(file => {
// console.log('directory',path.dirname(file))
if (path.dirname(file)==='.') file = repoPath+'/'+file
// console.log('file', file)
return read(file)
}))
.then((sets) => {
sets.forEach( set => {
if (set.isFulfilled) ignoreList.push(...set.value.toString().match(/.+/g))
else console.log('READ ERROR=> ',set.reason.Error)
})
// console.log('built list=====================', ignoreList)
return Promise.resolve(ignoreList)
})
.catch((err) => {
// only returned when something horrible is wrong
return Promise.reject(err)
})
}
export default ignore

54
src/read-lines.js Normal file
View file

@ -0,0 +1,54 @@
import { promisify } from 'util'
import { readFile, writeFile } from 'fs'
const read = promisify(readFile)
const write = promisify(writeFile)
// alternative with new promise experimental fs
// import fs from 'fs'
// const read = fs.promises.readFile
import settle from 'p-settle'
import path from 'path'
import logger from '@uci/logger'
let log = logger({ name: 'remote-code', file:'src/read-lines.js'})
// A promise helper function to return a list of paths to ignore from .npmignore, .gitignore, .rcignore
function readLines (files=[],dir) {
// console.log('additional files', files)
let list = []
if (!Array.isArray(files)) files=[files]
// each set in an the array is new line delimited set of ignore patterns
// settle returns array of error,value pairs
return settle(files.map(file => {
// console.log('directory',path.dirname(file))
if (path.dirname(file)==='.') file = dir+'/'+file
log.info({function:'readLines',file:file,msg:'reading a file of lines into array'})
return read(file)
}))
.then((sets) => {
sets.forEach( set => {
if (set.isFulfilled) list.push(...set.value.toString().match(/.+/g))
else log.warn({function:'readLines', error:set.reason, msg:' was unable to read file'})
})
return Promise.resolve(list)
})
.catch((err) => {
// only returned when something horrible is wrong
return Promise.reject(err)
})
}
// an ignore list should not be huge so can serailize at once
function writeLines (filePath,list) {
return write(filePath,list.join('\n'))
.then(() => {
log.info({function:'writeLines', file:filePath, msg:'wrote array to file of lines'})
})
.catch( err => {
log.fatal({function:'writeLines', error:err, msg:'unable to write array to file of lines'})
})
}
export default readLines
export { readLines, writeLines }

View file

@ -1,11 +1,15 @@
import { promisify } from 'util'
import readline from 'readline'
import { dirname } from 'path'
import to from 'await-to-js'
import { readFile } from 'fs'
const read = promisify(readFile)
import { readFile as readConfig } from 'fs-read-data'
import { Client } from 'ssh2'
import logger from '@uci/logger'
import Conf from 'conf'
let log = {} // declare module wide log to be set during construction
const read = promisify(readFile)
// ssh client is already an event emitter
class Ssh extends Client {
constructor(opts={}) {
@ -13,13 +17,35 @@ class Ssh extends Client {
this.opts = opts
log = logger({ name: 'remote-code', file:'src/ssh.js', class:'Ssh', id: opts.id })
this.ready = false
// additional options available
this.rpath = opts.rpath || '/opt' // will cd to this path upon connect
this.config = new Conf({projectName:'ssh-uci'})
this.configsDir = process.env.SSH_CONFIG_DIR || this.config.get('configsDir') || dirname(this.config.path)
console.log(this.configsDir)
this.rpath = opts.rpath || '/opt' // can be used to make all commands run in a particular remote path
}
async configure(opts,overwrite) {
let configPath
if (typeof opts ==='boolean' || !opts ) {
if (typeof opts ==='boolean') overwrite = opts
configPath = this.configsDir+'/'+this.config.get('defaultConfig')
opts = await readConfig(configPath)
}
if (typeof opts === 'string') {
configPath = (opts.indexOf('/') === -1)? this.configsDir+'/'+opts : opts
opts = await readConfig(configPath)
}
console.log('configPath',configPath)
console.log('opts',opts)
this.opts = overwrite ? Object.assign(this.opts,opts) : Object.assign(opts, this.opts)
}
async connect(copts = {}) {
const opts = Object.assign({},this.opts,copts) // merge any changes pasted via connect
// log.info('connection options ', opts)
if (opts.agentenv) opts.agent = process.env[opts.agentenv]
opts.readyTimeout = opts.readyTimeout | 5000 // default was 20 seconds...too long
opts.privateKey = opts.privateKey || opts.privateKeyPath ? await read(opts.privateKeyPath) : null
// log.info(opts.privateKeyPath, opts.privateKey)
@ -31,7 +57,7 @@ class Ssh extends Client {
resolve(`connected to ${this.opts.host}`)
})
this.on('error', (err) => {
log.info('connection error', err)
log.info({err:err, opts:this.opts, msg:'connection error'})
reject(err)
})
})
@ -39,10 +65,26 @@ class Ssh extends Client {
close() {
this.ready=false
log.info('closing ssh connection')
this.end()
}
async exec(command = 'ls -la', opts={}) {
async shell(command, opts={}) {
if (Object.prototype.toString.call(command) === '[object Object]') {
opts = command
if (!opts.cli) {
this.close()
let err = new Error('no terminal interactive shell must pass a command')
log.fatal({command:command, opts:opts, err:err, msg:'no terminal interactive shell must pass a command'})
return err
}
command = [] // a place to store all terminal commands
}
opts.shell=true
return this.exec(command,opts)
}
async exec(cmd = 'ls -la', opts={}) {
if (!this.ready) {
log.info('not connected, attempting connect first')
let [err] = await to(this.connect())
@ -50,27 +92,55 @@ class Ssh extends Client {
}
// log.info('executing command ', command)
const remote = this
let superexec = super.exec.bind(this)
const superexec = opts.shell? super.shell.bind(this) : super.exec.bind(this)
let command = opts.shell ? opts : cmd
return new Promise(function(resolve, reject) {
let reply=[]
let error=[]
let reply
let error
superexec(command, function(err, stream) {
console.log('command,cmd,opts',command,cmd,opts)
const done = () => {
if (error) remote.emit('remote:error',error)
resolve({command:command,cmds:cmd,reply:reply,error:error})
}
if (err) reject(err)
stream.on('finish', () => {
if (remote.opts.close || opts.close) {
log.info('closing connection')
remote.close
}
if (error.length!==0) remote.emit('client:error',error)
resolve({command:command,reply:reply,error:error})
// listerners
stream.once('close', () => {
if (remote.opts.close || opts.close || opts.cli) {
remote.on('end', done)
remote.close()
} else done()
}).on('data', function(data) {
reply += data.toString()
if (opts.cli) process.stdout.write(data.toString())
else log.info(`${remote.opts.host}$ ${data}`)
}).stderr.on('data', function(data) {
error += data.toString()
if (opts.cli) process.stderr.write(data.toString())
else log.info(`${remote.opts.host}:ERR $: ${data}`)
})
.on('data', function(data) {
reply.push(data)
log.info(`${remote.opts.host}$ ${data}`)
}).stderr.on('data', function(data) {
error.push(data)
log.info(`${remote.opts.host}:ERR$ ${data}`)
})
// alternate shell processing
if (opts.shell) {
if (opts.cli) { // terminal interactive shell sesssion
const cli= readline.createInterface({input:process.stdin, output:process.stdout})
cli.on('line', line => {
cmd.push(line)
console.log('storedcmds=>', cmd)
if(line.trim() === 'exit') {
console.log('ending stream')
cli.removeAllListeners()
cli.close()
stream.end(`${line}\n`)
} else {
console.log('not exit')
stream.write(`${line}\n`)
}
}) // end terminal listener
}
}
else {
stream.end(cmd) // one off shell command
} // end shell processing
}) // end super
}) // end Promise
}

View file

@ -1,61 +1,113 @@
const path = require('path')
const Rsync = require('rsync')
import { EventEmitter as Emitter } from 'events'
import path from 'path'
// import promisify from 'util'
import { readFile as readConfig } from 'fs-read-data'
import { exec as child } from 'child_process'
// import ignoreFiles from './ignore'
import logger from '@uci/logger'
let log = {} // declare module wide log to be set during construction
class Sync {
constructor(opts) {
this.options = opts
this.rsync = new Rsync()
.archive()
.delete()
.compress()
.dirs()
.exclude('node_modules/')
.exclude('.git/')
.source(path.join(opts.source, '/'))
.destination(`${opts.ssh.username}@${opts.ssh.host}:${path.join(opts.target)}`)
// can pass array of string of addition exlcudes - from a list likely
if (opts.excludes) {
this.rsync.exclude(opts.excludes)
}
if (opts.ssh.agent) {
this.rsync.shell(`ssh -p ${opts.ssh.port}`)
}
if (opts.ssh.keyfilePath) {
// This does NOT work with keys with pass phrase, use an ssh agent!
this.rsync.shell(`ssh -i ${opts.ssh.keyfilePath} -p ${opts.ssh.port}`)
}
if (opts.ssh.password) {
this.rsync.shell(`sshpass -p ${opts.ssh.password} ssh -o StrictHostKeyChecking=no -l ${opts.ssh.username}`)
}
console.log(this.rsync.command())
this.syncInProgress = false
return this
class Sync extends Emitter {
constructor(opts = {}) {
super()
log = logger({ name: 'remote-code', file:'src/sync.js', class:'Sync'})
this.opts = opts
// this.opts.more = this.opts.more || []
// this.opts.ssh = this.opts.ssh || []
this.jobOpts = {}
}
addStdOutStream(stream) {
this.stdOutStream = stream
return this
async configure(opts,overwrite) {
if (typeof opts === 'string') opts = await readConfig(opts) // opts here is path to configuration file
this.opts = overwrite ? Object.assign(this.opts,opts) : Object.assign(opts, this.opts)
}
addStdErrStream(stream) {
this.stdErrStream = stream
return this
addOpt(option,value) {
this.opts.more = this.opts.more || []
this.opts.more.push(`${(option.length > 1)?'--':'-'}${option} ${value}`)
log.info (this.opts.more)
}
execute() {
this.syncInProgress = true
addSshOpt(option,value) {
this.opts.ssh = this.opts.ssh || []
this.opts.ssh.push(`${(option.length > 1)?'--':'-'}${option} ${value}`)
log.info (this.opts.ssh)
}
command(src, dest) {
this.opts.src = src || this.opts.src || process.cwd()
this.opts.dest = dest || this.opts.dest || `/opt/${process.cwd().match(/([^/]*)\/*$/)[1]}`
this.cmd=`rsync ${this._opts} ${this._ssh()} ${this.opts.src}/ ${this.opts.dest}`
// this.cmd = JSON.stringify(this.opts)
return this.cmd
}
_ssh() {
if (!(this.opts.username && this.opts.host)) return ''
this.opts.dest = `${this.opts.username}@${this.opts.host}:${this.opts.dest}`
if (this.opts.keyFilePath) this.addSshOpt('i',this.opts.keyFilePath)
if (this.opts.port) this.addSshOpt('p',this.opts.port)
if (!this.opts.ssh) return ''
return `-e "ssh ${this.opts.ssh.join(' ')}"`
}
_opts() {
if (typeof opts.mirror==='boolean' && !opts.mirror)
// if ssh agent is not available in child process can make it so
_agent(path) {
// this.jobOpts.env
}
// returns a promise when job is done
execute(src,dest) {
const sync = this
let reply=[]
let error=[]
sync.emit('busy')
return new Promise((resolve, reject) => {
this.rsync.execute((err, resCode) => {
if (err) {
return reject(err)
}
this.syncInProgress = false
resolve(resCode)
},
this.stdOutStream,
this.stdErrStream)
// may need to be sure that child has agent socket envirnoment variable
if (this.opts.agent) this._agent(this.opts.agent)
const job = child(this.command(src,dest), this.jobOpts)
job.on('error', (err) => {
log.fatal({err:err, opts:sync.opts, msg:'failed to spawn rsync job'})
reject(err)
})
job.on('close', (code) => {
log.info(`sync job exited with code ${code}`)
if (error.length!==0) sync.emit('remote:error',error)
sync.emit('done',{command:sync.cmd, reply:reply})
resolve({command:sync.cmd,reply:reply,error:error})
})
job.stdout.on('data', (data) => {
log.info({remote:sync.opts.host, console:data, msg:'console data from remote'})
reply.push(data)
})
job.stderr.on('data', (data) => {
log.warn({remote:sync.opts.host, error:data, msg:'error from remote'})
error.push(data)
})
})
}
}// end Class Sync
export default Sync
function isPlainObject (obj) {
return Object.prototype.toString.call(obj) === '[object Object]'
}
module.exports = Sync
function escapeSpaces (str) {
if (typeof str === 'string') {
return str.replace(/\b\s/g, '\\ ')
} else {
return path
}
}

View file

@ -1 +0,0 @@
wtf/

6
test/config/ssh.yaml Normal file
View file

@ -0,0 +1,6 @@
host: switches.kebler.net
username: sysadmin
# agent: /run/user/1000/ssh-agent.socket
# agentenv: SSH_AUTH_SOCK
#privateKeyPath: /home/david/.ssh/privatekeys/sysadmin.kebler.net
#passphrase: '51535560'

6
test/config/sync.yaml Normal file
View file

@ -0,0 +1,6 @@
host: giskard.kebler.net
#port: 33
username: sysadmin
keyFilePath: '~/.ssh/keyfile'
src: ./test/repo
dest: /opt/repo

View file

@ -1,64 +0,0 @@
'use strict'
import ignore from '../src/ignore'
import { expect } from 'chai'
import { it } from 'mocha'
// pause = require('@uci/utils').pPause
describe (
'Build Ignore Array',
function () {
// hooks()
buildIgnoreList()
// someothertests()
})
//****************** TESTS **********************
function buildIgnoreList() {
it('==> can create array for multiple files of ignore lines', async function () {
const shouldbe = [ 'tests/',
'test/',
'*.test.js',
'testing/',
'example/',
'/node_modules/',
'/coverage/',
'foo/**',
'bar/*.js' ]
let result = await ignore(__dirname+'/repo')
//console.log('returned to test',result, shouldbe)
expect(result, 'list build failed').to.deep.equal(shouldbe)
})
it('==> can create array with additional passed in file and relative repo path', async function () {
const shouldbe = [ 'tests/',
'test/',
'*.test.js',
'testing/',
'example/',
'/node_modules/',
'/coverage/',
'foo/**',
'bar/*.js',
'bah/*/*.js'
]
let result = await ignore('./test/repo','.testignore')
//console.log('returned to test',result, shouldbe)
expect(result, 'list build failed').to.deep.equal(shouldbe)
})
it('==> can handle file in other than repo path', async function () {
const shouldbe = [ 'tests/',
'test/',
'*.test.js',
'testing/',
'example/',
'/node_modules/',
'/coverage/',
'foo/**',
'bar/*.js',
'wtf/'
]
let result = await ignore(__dirname+'/repo','./test/anotherignorefile')
//console.log('returned to test',result, shouldbe)
expect(result, 'list build failed').to.deep.equal(shouldbe)
})
}

View file

@ -0,0 +1,54 @@
// let ignoreFiles = ['.npmignore','.gitignore']
import { readLines, writeLines } from '../src/read-lines'
import chai from 'chai'
import assertArrays from 'chai-arrays'
import { it } from 'mocha'
chai.use(assertArrays)
const expect = chai.expect
describe (
'Read a File of Lines to Array and vice versa',
function () {
readList()
writeList()
})
//****************** TESTS **********************
function readList() {
it('==> can read one or more files (no order) each line as element in an array with common directory', async function () {
const shouldbe = [ 'tests/',
'test/',
'*.test.js',
'testing/',
'example/',
'/node_modules/',
'/coverage/' ]
let result = await readLines(['.gitignore','.npmignore'],__dirname+'/repo')
expect(result, 'list build failed').to.be.containingAllOf(shouldbe)
})
it('==> can read two files one relative the other absolute', async function () {
const shouldbe = [ 'tests/',
'test/',
'*.test.js',
'testing/',
'example/',
'/node_modules/',
'/coverage/' ]
let result = await readLines(['./test/repo/.gitignore',__dirname+'/repo/.npmignore'])
expect(result, 'list build failed').to.be.containingAllOf(shouldbe)
})
}
function writeList() {
it('==> can write an array items as lines in a file', async function () {
const shouldbe = await readLines(['.gitignore','.npmignore'],__dirname+'/repo')
await writeLines(__dirname+'/repo/combined.list',shouldbe)
const result = await readLines(['combined.list'],__dirname+'/repo')
expect(result, 'list build failed').to.be.containingAllOf(shouldbe)
})
}

7
test/repo/combined.list Normal file
View file

@ -0,0 +1,7 @@
/node_modules/
/coverage/
tests/
test/
*.test.js
testing/
example/

View file

@ -2,18 +2,16 @@ import Ssh from '../src/ssh'
import to from 'await-to-js'
import { expect } from 'chai'
import { it } from 'mocha'
import read from 'read-yaml-file'
import logger from '@uci/logger'
// pause = require('@uci/utils').pPause
describe('SSH Class Testing ',async ()=> {
let remote
let log
before(async () => {
log = logger({ name: 'remote-code', test:'/test/ssh.test.js', class:'Ssh',file:'src/ssh.js', id: 'testing' })
let opts = await read('./test/ssh.yaml')
remote = new Ssh(opts)
log = logger({ name: 'remote-code', test:'/test/ssh.test.js', class:'Ssh', file:'src/ssh.js', id: 'testing' })
remote = new Ssh()
await remote.configure('./test/config/ssh') // will pick up config file format
log.info(`making connection to ${remote.opts.host}`)
let [err] = await to(remote.connect())
if (err) {
@ -33,7 +31,7 @@ describe('SSH Class Testing ',async ()=> {
log.info('error running command aborting test', err)
return
}
log.info(`result of remote command ${res.command} => ${res.reply.toString().trim()}`)
log.info({remoteHost:remote.opts.host, reply:res.reply.toString().trim(), err:err, cmd:res.command, msg:`remote command completed ${res.command}`})
expect(res.reply.toString().trim(), 'test failed').to.equal('/opt')
})

View file

@ -1,5 +0,0 @@
host: switches.kebler.net
username: sysadmin
# agent: /run/user/1000/ssh-agent.socket
privateKeyPath: /home/david/.ssh/privatekeys/sysadmin.kebler.net
passphrase: '51535560'

48
test/sync.test.js Normal file
View file

@ -0,0 +1,48 @@
import Sync from '../src/sync'
import to from 'await-to-js'
import { expect } from 'chai'
import { it } from 'mocha'
import logger from '@uci/logger'
// pause = require('@uci/utils').pPause
describe('Sync Class Testing ',async ()=> {
let sync
let log
before(async () => {
log = logger({ name: 'remote-code', test:'/test/sync.test.js', class:'sync',file:'src/sync.js', id: 'testing' })
sync = new Sync()
await sync.configure('./test/repo/config')
log.info({cmd:sync.command(), msg:'Rsync Command that will Run'})
// log.info(`making connection to ${remote.opts.host}`)
// log.info('ready for testing')
})
// after(async () => {
// remote.close()
// })
it('can sync a directory' , async function () {
// let [err,res] = await to(remote.exec('cd /opt && pwd'))
// if (err) {
// log.info('error running command aborting test', err)
// return
// }
// log.info(`result of remote command ${res.command} => ${res.reply.toString().trim()}`)
expect('test', 'test failed').to.equal('/opt')
})
})
// function hooks(remote) {
//
// // beforeEach(async() => {
// // await someasyncfunctiontodobeforeeachtest()
// // })
//
// // after(async() => {
// // await someasyncfunctiontodoaftereeachtest()
// // })
//
// }