From 11ca82e084e70eab3cd9e0540a03d1e5e69c2739 Mon Sep 17 00:00:00 2001 From: David Kebler Date: Tue, 29 Jan 2019 12:28:37 -0800 Subject: [PATCH] Initial commit (a fork of sorts from remote-code https://www.npmjs.com/package/remote-code) Started refactoring the 4 primary classes Added ignore list generator from various default ignore files. wrote corresponding test -- passing --- .eslintrc.js | 37 +++++++++++++ .gitignore | 2 + .npmignore | 5 ++ package.json | 39 +++++++++++++ readme.md | 21 +++++++ src/ignore.js | 39 +++++++++++++ src/index.js | 8 +++ src/remote-code.js | 123 +++++++++++++++++++++++++++++++++++++++++ src/ssh.js | 97 ++++++++++++++++++++++++++++++++ src/sync.js | 61 ++++++++++++++++++++ src/watcher.js | 41 ++++++++++++++ src/watcher.test.js | 49 ++++++++++++++++ test/anotherignorefile | 1 + test/ignore.test.js | 64 +++++++++++++++++++++ test/repo/.gitignore | 2 + test/repo/.npmignore | 5 ++ test/repo/.rcignore | 2 + test/repo/.testignore | 1 + 18 files changed, 597 insertions(+) create mode 100644 .eslintrc.js create mode 100644 .gitignore create mode 100644 .npmignore create mode 100644 package.json create mode 100644 readme.md create mode 100644 src/ignore.js create mode 100644 src/index.js create mode 100644 src/remote-code.js create mode 100644 src/ssh.js create mode 100644 src/sync.js create mode 100644 src/watcher.js create mode 100644 src/watcher.test.js create mode 100644 test/anotherignorefile create mode 100644 test/ignore.test.js create mode 100644 test/repo/.gitignore create mode 100644 test/repo/.npmignore create mode 100644 test/repo/.rcignore create mode 100644 test/repo/.testignore diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000..49bac18 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,37 @@ +module.exports = { + "ecmaFeatures": { + "modules": true, + "spread" : true, + "restParams" : true + }, + // "plugins": [ + // "unicorn" + // ], + "env": { + "es6": true, + "node": true, + "mocha": true + }, + "parserOptions": { + "ecmaVersion": 2017, + "sourceType": "module" + }, + "extends": "eslint:recommended", + "rules": { + "indent": [ + "error", + 2 + ], + // "unicorn/no-array-instanceof": "error", + "no-console": 0, + "semi": ["error", "never"], + "linebreak-style": [ + "error", + "unix" + ], + "quotes": [ + "error", + "single" + ] + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e61051f --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/node_modules/ +/coverage/ diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..02078d5 --- /dev/null +++ b/.npmignore @@ -0,0 +1,5 @@ +tests/ +test/ +*.test.js +testing/ +example/ diff --git a/package.json b/package.json new file mode 100644 index 0000000..c35a716 --- /dev/null +++ b/package.json @@ -0,0 +1,39 @@ +{ + "name": "@uci/remote-code", + "version": "0.0.1", + "description": "module to copy, maintain, and launch hardware modules on other machines", + "main": "src/index.js", + "scripts": { + "testd": "./node_modules/.bin/mocha -r esm --watch --timeout 30000 || exit 0", + "test": "./node_modules/.bin/mocha -r esm --timeout 30000 || exit 0", + "testibc": "istanbul cover ./node_modules/.bin/_mocha test/ --report lcovonly -- -R spec --recursive && codecov || true" + }, + "author": "David Kebler", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/uCOMmandIt/uci-remote-code.git" + }, + "keywords": [ + "node.js", + "I2C", + "raspberryPi" + ], + "bugs": { + "url": "https://github.com/uCOMmandIt/uci-remote-code/issues" + }, + "homepage": "https://github.com/uCOMmandIt/uci-remote-code#readme", + "dependencies": { + "chokidar": "^2.0.4", + "p-settle": "^2.1.0", + "rsyncwrapper": "^3.0.1", + "ssh2": "^0.8.2" + }, + "devDependencies": { + "chai": "^4.2.0", + "codecov": "^3.1.0", + "esm": "^3.1.4", + "istanbul": "^0.4.5", + "mocha": "^5.x" + } +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..83b6323 --- /dev/null +++ b/readme.md @@ -0,0 +1,21 @@ +# uCOMmandIt Template Package Repository + + +[![Build Status](https://img.shields.io/travis/uCOMmandIt/uci-pkg-template.svg?branch=master)](https://travis-ci.org/uCOMmandIt/uci-pkg-template) +[![Inline docs](http://inch-ci.org/github/uCOMmandIt/uci-pkg-template.svg?branch=master)](http://inch-ci.org/github/uCOMmandIt/uci-pkg-template) +[![Dependencies](https://img.shields.io/david/uCOMmandIt/uci-pkg-template.svg)](https://david-dm.org/uCOMmandIt/uci-pkg-template) +[![devDependencies](https://img.shields.io/david/dev/uCOMmandIt/uci-pkg-template.svg)](https://david-dm.org/uCOMmandIt/uci-pkg-template?type=dev) +[![codecov](https://img.shields.io/codecov/c/github/uCOMmandIt/uci-pkg-template/master.svg)](https://codecov.io/gh/uCOMmandIt/uci-pkg-template) + +Clone this to get a quick start on a new uci package. It has all the testing ready to go with Travis-CI and code coverage. All the readme badges are included as well + +Clone it for as a starting place for your own package! + +You'll need codecov and travis-ci accounts + +##Steps + +1. Clone repo +2. Edit package.json +3. Edit badge urls above changing to repo path +4. Start Coding diff --git a/src/ignore.js b/src/ignore.js new file mode 100644 index 0000000..e6692a8 --- /dev/null +++ b/src/ignore.js @@ -0,0 +1,39 @@ +import settle from 'p-settle' +import { promisify } from 'util' +import { readFile } from 'fs' +import path from 'path' +const read = promisify(readFile) + +// A helper function to return a list of paths to ignore from .npmignore, .gitignore, .rcignore +function ignore (repoPath,files) { + // console.log('additional files', files) + let ignoreList = [] + let ignoreFiles = ['.npmignore','.gitignore','.rcignore'] + if (Array.isArray(files)) { + ignoreFiles=[...ignoreFiles,...files] + } else { + if (files) ignoreFiles.push(files) + } + + // each set in an the array is new line delimited set of ignore patterns + return settle(ignoreFiles.map(file => { + // console.log('directory',path.dirname(file)) + if (path.dirname(file)==='.') file = repoPath+'/'+file + // console.log('file', file) + return read(file) + })) + .then((sets) => { + sets.forEach( set => { + if (set.isFulfilled) ignoreList.push(...set.value.toString().match(/.+/g)) + else console.log('READ ERROR=> ',set.reason.Error) + }) + // console.log('built list=====================', ignoreList) + return Promise.resolve(ignoreList) + }) + .catch((err) => { + // only returned when something horrible is wrong + return Promise.reject(err) + }) +} + +export default ignore diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..aadf982 --- /dev/null +++ b/src/index.js @@ -0,0 +1,8 @@ +import RemoteCode from './remote-code' +import Ssh from './ssh' +import Sync from './sync' +import Watcher from './watcher' +import ignore from './ignore' + +export { RemoteCode, Ssh, Sync, Watcher, ignore } +export default RemoteCode diff --git a/src/remote-code.js b/src/remote-code.js new file mode 100644 index 0000000..46cc93f --- /dev/null +++ b/src/remote-code.js @@ -0,0 +1,123 @@ +import { EventEmitter as Emitter } from 'events' +import stream from 'stream' +import Watcher from './watcher' +import Ssh from './ssh' +import Sync from './lib/sync' + +const devNull = new stream.Writable() +devNull._write = () => null + +class RemoteCode { + constructor(opts = {}) { + // Set defaults + // a id emitted with every event emit that can identify which remote code emitted + this.id = opts.id + this.watch = opts.watch || false + this.options.ssh.keepaliveInterval = opts.ssh.keepaliveInterval || 500 + this.options.ssh.readyTimeout = opts.ssh.readyTimeout || 2000 + this.options.ssh.port = opts.ssh.port || 22 + this.options.source = opts.source || '.' + this.options.install = opts.install || 'yarn' + this.options.install = opts.registry ? `${this.options.install} --registry ${opts.registry}` : this.options.install + this.options.start = opts.start || 'nodemon .' + if (!(opts.ssh.keyfilePath || opts.ssh.password)) { + this.options.ssh.agent = process.env[opts.ssh.agent] || process.env.SSH_AUTH_SOCK || process.env.SSH_AGENT_SOCK + if (!this.options.ssh.agent) { + return new Error('no ssh authentification method provided') + } + } + this.ssh = new Ssh() + this.verbose = opts.verbose || false + this.stdout = opts.stdout instanceof stream.Writable ? opts.stdout : new stream.Writable() + this.stderr = opts.stderr instanceof stream.Writable ? opts.stderr : new stream.Writable() + this.watcher = new Watcher(this.options) + this.sync = new Sync(this.options) + .addStdOutStream(this.stdout) + .addStdErrStream(this.stderr) + } + + + + _getStdOut() { + if (this.verbose) { + return this.stdout + } + return devNull + } + + _getStdErr() { + if (this.verbose) { + return this.stderr + } + return devNull + } + + sync() { + this.emit('sync',{}) + return this.sync.execute() + } + + watch() { + this.watcher.start() + const watchEmitter = this.watcher.getEventEmitter() + watchEmitter.on('sync', () => { + this.syncCode() + }) + watchEmitter.on('install', () => { + return this.install() + .then(() => { + this.ssh.liveReload.send('rs') + }) + }) + return this.emitter + } + + async init() { + this.emitter.emit('start') + const sshSettings = this.options.ssh + return Promise.all([this.syncCode(), this.watch()]) + .then(() => this.install()) + .then(() => this.ssh.liveReload.connect(sshSettings)) + .then(() => { + this.emitter.emit('nodemon', 'start') + this.ssh.liveReload.send(`cd ${this.options.target} && ${this.options.start}`) + }) + .catch(this._abort.bind(this)) + } + + // execute a single command and then resolve + execute(cmd, stdout, stderr) { + console.log('remote codes exec', cmd) + this.emitter.emit('exec', cmd) + const ssh = new Ssh() + const result = ssh.exec(this.options.ssh, cmd, stdout, stderr) + return result + } + + async install() { + this.emit('install', 'triggered') + if (!this.installInProgress) { + this.emitter.emit('install', 'started') + console.log('calling execute from install') + return this.execute(`cd ${this.options.target} && ${this.options.install}`, this._getStdOut(), this._getStdErr()) + .then(res => { + this.emit('install', 'done', res) + this.installInProgress = false + return res + }) + } + return Promise.resolve() + } + + close() { + this.emitter.emit('close') + return Promise.all([this.watcher.close(), + this.ssh.liveReload.close()]) + } + + _abort(err) { + this.emitter.emit('error', err) + } +} + +export default RemoteCode diff --git a/src/ssh.js b/src/ssh.js new file mode 100644 index 0000000..c7929bc --- /dev/null +++ b/src/ssh.js @@ -0,0 +1,97 @@ +const fs = require('fs') +const EventEmitter = require('events').EventEmitter +const ssh = require('ssh2') + +const Client = ssh.Client + +class Ssh extends Client { + constructor() { + super() + this._emitter = new EventEmitter() + this._ignoreStdOut = [] + return this.init() + } + + init() { + const conn = this + conn.on('ready', () => { + conn.shell((err, stream) => { + if (err) { + throw err + } + this.stream = stream + this._emit('connect', 'Connection established') + stream.on('close', () => { + this._emit('close', 'Connection closed') + conn.end() + }) + stream.on('data', data => this._emit('data', data)) + stream.stderr.on('data', data => this._emit('error', data)) + }) + }) + return this + } + + connect(opts = {}) { + opts.privateKey = opts.keyfilePath ? fs.readFileSync(opts.keyfilePath) : null + super.connect(opts) + return new Promise(resolve => { + this.getEventEmitter().on('connect', () => resolve(this)) + }) + } + + getEventEmitter() { + return this._emitter + } + + _emit(...args) { + this._emitter.emit(...args) + } + + send(line) { + this._ignoreStdOut.push(line) + if (this.stream.writable) { + this.stream.write(`${line}\n`) + } + } + + close() { + this.end() + this.stream = null + return this + } + + // Supercedes ssh2.exec + // execute a single command on the server and close & resolve + // uses interactive shell to get stdout/stderr back + // stdout, stderr are streams + exec(opts, command = 'uptime', stdout, stderr) { + let data = '' + console.log('executing', opts, command) + return new Promise((resolve, reject) => { + this + .on('ready', () => { + this.shell((err, stream) => { + stream + .on('close', () => { + this.end() + resolve(data) + }) + .on('data', s => { + data += s + stdout.write(s) + }) + .stderr.on('data', s => { + data += s + stderr.write(s) + }) + stream.end(command + '\nexit\n') + }) + }) + .on('error', e => reject(e)) + .connect(opts) + }) + } +} + +module.exports = Ssh diff --git a/src/sync.js b/src/sync.js new file mode 100644 index 0000000..823ccd2 --- /dev/null +++ b/src/sync.js @@ -0,0 +1,61 @@ +const path = require('path') +const Rsync = require('rsync') + +class Sync { + constructor(opts) { + this.options = opts + this.rsync = new Rsync() + .archive() + .delete() + .compress() + .dirs() + .exclude('node_modules/') + .exclude('.git/') + .source(path.join(opts.source, '/')) + .destination(`${opts.ssh.username}@${opts.ssh.host}:${path.join(opts.target)}`) + // can pass array of string of addition exlcudes - from a list likely + if (opts.excludes) { + this.rsync.exclude(opts.excludes) + } + if (opts.ssh.agent) { + this.rsync.shell(`ssh -p ${opts.ssh.port}`) + } + if (opts.ssh.keyfilePath) { + // This does NOT work with keys with pass phrase, use an ssh agent! + this.rsync.shell(`ssh -i ${opts.ssh.keyfilePath} -p ${opts.ssh.port}`) + } + if (opts.ssh.password) { + this.rsync.shell(`sshpass -p ${opts.ssh.password} ssh -o StrictHostKeyChecking=no -l ${opts.ssh.username}`) + } + console.log(this.rsync.command()) + this.syncInProgress = false + return this + } + + addStdOutStream(stream) { + this.stdOutStream = stream + return this + } + + addStdErrStream(stream) { + this.stdErrStream = stream + return this + } + + execute() { + this.syncInProgress = true + return new Promise((resolve, reject) => { + this.rsync.execute((err, resCode) => { + if (err) { + return reject(err) + } + this.syncInProgress = false + resolve(resCode) + }, + this.stdOutStream, + this.stdErrStream) + }) + } +} + +module.exports = Sync diff --git a/src/watcher.js b/src/watcher.js new file mode 100644 index 0000000..f7a3ad8 --- /dev/null +++ b/src/watcher.js @@ -0,0 +1,41 @@ +import { EventEmitter as Emitter } from 'events' +import path from 'path' +import chokidar from 'chokidar' + +class Watcher extends Emitter { + constructor(options) { + super() + // pass in ignores + const opts = { + ignored: '**/node_modules/**', + ignoreInitial: true + } + const watcher = chokidar.watch(options.source, opts) + this.watcher = watcher + } + + start() { + const handler = (type, f) => { + const fname = path.basename(f) + if ( fname.toLowerCase() === 'package.json') + if (type !=='change') { + this.emit('error',new Error('package.json was added or removed, ignoring sync and reinstall')) + return + } else{ + this.emit('install', f) + } + this.emit('sync', f) + } // end handler + this.watcher + .on('add', handler.bind(this, 'add')) + .on('change', handler.bind(this, 'change')) + .on('unlink', handler.bind(this, 'remove')) + } + + stop() { + this.watcher.close() + } + +} + +export default Watcher diff --git a/src/watcher.test.js b/src/watcher.test.js new file mode 100644 index 0000000..7b589c1 --- /dev/null +++ b/src/watcher.test.js @@ -0,0 +1,49 @@ +import events from 'events'; +import test from 'ava'; +import td from 'testdouble'; + +const chokidarStub = {}; + +td.replace('chokidar', chokidarStub); +const Fn = require('./watcher'); + +function setup(stubs = {}) { + chokidarStub.watch = stubs.watch || (() => new events.EventEmitter()); + return new Fn({source: '.'}); +} + +test('should emit "sync" if file was added/changed/deleted', t => { + t.plan(3); + const fn = setup(); + fn.start(); + fn.getEventEmitter().on('sync', () => t.pass('file change detected')); + fn.watcher.emit('add', 'file.txt'); + fn.watcher.emit('change', 'file.txt'); + fn.watcher.emit('unlink', 'file.txt'); +}); + +test('should emit "install" if "package.json" or "yarn.lock" was added/changed/deleted', t => { + t.plan(6); + const fn = setup(); + fn.start(); + fn.getEventEmitter().on('install', () => t.pass('file change detected')); + fn.watcher.emit('add', 'package.json'); + fn.watcher.emit('change', 'package.json'); + fn.watcher.emit('unlink', 'package.json'); + fn.watcher.emit('add', 'yarn.lock'); + fn.watcher.emit('change', 'yarn.lock'); + fn.watcher.emit('unlink', 'yarn.lock'); +}); + +test('should emit "sync" if "package.json" or "yarn.lock" was added/changed/deleted', t => { + t.plan(6); + const fn = setup(); + fn.start(); + fn.getEventEmitter().on('sync', () => t.pass('file change detected')); + fn.watcher.emit('add', 'package.json'); + fn.watcher.emit('change', 'package.json'); + fn.watcher.emit('unlink', 'package.json'); + fn.watcher.emit('add', 'yarn.lock'); + fn.watcher.emit('change', 'yarn.lock'); + fn.watcher.emit('unlink', 'yarn.lock'); +}); diff --git a/test/anotherignorefile b/test/anotherignorefile new file mode 100644 index 0000000..b3ecd80 --- /dev/null +++ b/test/anotherignorefile @@ -0,0 +1 @@ +wtf/ diff --git a/test/ignore.test.js b/test/ignore.test.js new file mode 100644 index 0000000..1cbf9e2 --- /dev/null +++ b/test/ignore.test.js @@ -0,0 +1,64 @@ +'use strict' + +import ignore from '../src/ignore' +import { expect } from 'chai' +import { it } from 'mocha' +// pause = require('@uci/utils').pPause + +describe ( + 'Build Ignore Array', + function () { + // hooks() + buildIgnoreList() + // someothertests() + }) + +//****************** TESTS ********************** +function buildIgnoreList() { + it('==> can create array for multiple files of ignore lines', async function () { + const shouldbe = [ 'tests/', + 'test/', + '*.test.js', + 'testing/', + 'example/', + '/node_modules/', + '/coverage/', + 'foo/**', + 'bar/*.js' ] + let result = await ignore(__dirname+'/repo') + //console.log('returned to test',result, shouldbe) + expect(result, 'list build failed').to.deep.equal(shouldbe) + }) + it('==> can create array with additional passed in file and relative repo path', async function () { + const shouldbe = [ 'tests/', + 'test/', + '*.test.js', + 'testing/', + 'example/', + '/node_modules/', + '/coverage/', + 'foo/**', + 'bar/*.js', + 'bah/*/*.js' + ] + let result = await ignore('./test/repo','.testignore') + //console.log('returned to test',result, shouldbe) + expect(result, 'list build failed').to.deep.equal(shouldbe) + }) + it('==> can handle file in other than repo path', async function () { + const shouldbe = [ 'tests/', + 'test/', + '*.test.js', + 'testing/', + 'example/', + '/node_modules/', + '/coverage/', + 'foo/**', + 'bar/*.js', + 'wtf/' + ] + let result = await ignore(__dirname+'/repo','./test/anotherignorefile') + //console.log('returned to test',result, shouldbe) + expect(result, 'list build failed').to.deep.equal(shouldbe) + }) +} diff --git a/test/repo/.gitignore b/test/repo/.gitignore new file mode 100644 index 0000000..e61051f --- /dev/null +++ b/test/repo/.gitignore @@ -0,0 +1,2 @@ +/node_modules/ +/coverage/ diff --git a/test/repo/.npmignore b/test/repo/.npmignore new file mode 100644 index 0000000..02078d5 --- /dev/null +++ b/test/repo/.npmignore @@ -0,0 +1,5 @@ +tests/ +test/ +*.test.js +testing/ +example/ diff --git a/test/repo/.rcignore b/test/repo/.rcignore new file mode 100644 index 0000000..c3dfa01 --- /dev/null +++ b/test/repo/.rcignore @@ -0,0 +1,2 @@ +foo/** +bar/*.js diff --git a/test/repo/.testignore b/test/repo/.testignore new file mode 100644 index 0000000..6d2e0a4 --- /dev/null +++ b/test/repo/.testignore @@ -0,0 +1 @@ +bah/*/*.js