in process of refactoring - just a backup commit

master
Kebler Network System Administrator 2021-07-26 12:23:11 -07:00
parent 281b31ecb5
commit 8e6d15ca49
23 changed files with 328 additions and 200 deletions

10
.babelrc Executable file
View File

@ -0,0 +1,10 @@
{
"presets": [
[
"@babel/preset-env",
{
"shippedProposals": true
}
]
]
}

View File

@ -1,37 +0,0 @@
module.exports = {
"ecmaFeatures": {
"modules": true,
"spread" : true,
"restParams" : true
},
// "plugins": [
// "unicorn"
// ],
"env": {
"es6": true,
"node": true,
"mocha": true
},
"parserOptions": {
"ecmaVersion": 2017,
"sourceType": "module"
},
"extends": "eslint:recommended",
"rules": {
"indent": [
"error",
2
],
// "unicorn/no-array-instanceof": "error",
"no-console": 0,
"semi": ["error", "never"],
"linebreak-style": [
"error",
"unix"
],
"quotes": [
"error",
"single"
]
}
}

21
.eslintrc.yml Executable file
View File

@ -0,0 +1,21 @@
env:
node: true
es2021: true
mocha: true
extends:
- standard
parserOptions:
ecmaVersion: 12
sourceType: module
rules:
indent:
- error
- 2
no-console: 0
semi:
- error
- never
quotes:
- error
- single
parser: '@babel/eslint-parser'

View File

@ -1,14 +1,14 @@
#!/usr/bin/env node
import Sync from '../src/sync'
import Sync from '../src/sync.js'
import to from 'await-to-js'
// import logger from '@uci-utils/logger'
import yargs from 'yargs'
import Yargs from 'yargs'
import readlines from '@uci-utils/read-lines'
let jobname; let command
console.log(process.argv)
let args = yargs
const args = Yargs(process.argv.slice(2))
.boolean('w')
.alias('w','watch')
.boolean('v')
@ -23,7 +23,7 @@ if (args._.length ===1) {
jobname = args._[0]
command = args._[1]
}
console.log(jobname, command)
console.log('jobname',jobname, 'command',command)
;
(async () => {

6
example/example.js Normal file → Executable file
View File

@ -1,4 +1,4 @@
import Sync from '../src/sync'
import Sync from '../src/sync.js'
import onDeath from 'ondeath'
(async () => {
@ -6,8 +6,8 @@ import onDeath from 'ondeath'
await sync.runJob('remote')
// await sync.loadJob('local')
// sync.watch('on')
// console.log('ready and waiting')
sync.watch('on')
console.log('ready and waiting')
onDeath( () => {
console.log('\nHe\'s dead Jim')

24
example/jobs/remote.yaml Normal file → Executable file
View File

@ -1,23 +1,25 @@
source: ./example/source/ # be sure to add trailing / to avoid making a subdirectory under destination
destination: /opt/example
flags: 'av'
# be sure to add trailing / to avoid making a subdirectory under destination
source: ./example/source/
destination: example
flags: "av"
runOpts:
cli: true
cli: true
excludeFrom:
- .gitignore
- example/source/.npmignore
- .gitignore
- example/source/.npmignore
exclude:
- .gitignore
- .npmignore
- .gitignore
- .npmignore
set:
- delete
- delete-excluded
- stats
watch: # true
watch: # true
wait: 200
immediate: true
ssh:
host: relays.kebler.net
username: sysadmin
host: trantor.kebler.net
username: rbackup
privateKeyPath: /mnt/AllData/secure/privatekeys/xfer.kebler.net
# quiet: true
# additional: ''

22
nodemon.json Normal file → Executable file
View File

@ -1,3 +1,21 @@
{
"ext": "js, yaml"
}
"ignoreRoot": [
".git"
],
"watch": [
"node_modules/@uci/",
"node_modules/@uci-utils/",
"readme/",
"src/",
"index.js",
"examples",
"test/",
"examples/"
],
"ignore": [
"README.md",
"package*.json",
"test/files/"
],
"ext": "js,json,yaml,yml,md"
}

107
package.json Normal file → Executable file
View File

@ -1,52 +1,59 @@
{
"name": "@uci-utils/sync",
"version": "0.0.10",
"description": "module to copy, maintain, and launch hardware modules on other machines",
"main": "src/index.js",
"bin": {
"syncu": "./bin/sync-cli"
},
"scripts": {
"sync": "node -r esm ./bin/sync",
"example": "node -r esm ./example/example",
"test": "./node_modules/.bin/mocha -r esm --timeout 30000",
"testd": "UCI_ENV=dev ./node_modules/.bin/nodemon --exec './node_modules/.bin/mocha -r esm --timeout 30000'",
"testdd": "UCI_LOG_LEVEL='trace' npm run testd",
"testde": "UCI_LOG_LEVEL='warn' npm run testd",
"testl": "UCI_ENV=pro UCI_LOG_PATH=./test/test.log 0 npm run test || exit 0"
},
"author": "David Kebler",
"license": "MIT",
"repository": {
"type": "git",
"url": "git+https://github.com/uCOMmandIt/uci-remote-code.git"
},
"keywords": [
"node.js",
"I2C",
"raspberryPi"
],
"bugs": {
"url": "https://github.com/uCOMmandIt/uci-remote-code/issues"
},
"homepage": "https://github.com/uCOMmandIt/uci-remote-code#readme",
"dependencies": {
"@uci-utils/class-merge": "^1.0.3",
"@uci-utils/logger": "^0.0.16",
"@uci-utils/read-lines": "^0.2.2",
"@uci-utils/watcher": "^0.2.4",
"await-to-js": "^2.1.1",
"conf": "^5.0.0",
"debounce-fn": "^3.0.1",
"fs-read-data": "^1.0.4",
"path-exists": "^4.0.0",
"yargs": "^13.3.0"
},
"devDependencies": {
"chai": "^4.2.0",
"esm": "^3.2.25",
"mocha": "^6.x",
"nodemon": "^1.19.1",
"ondeath": "^1.0.0"
}
"main": "src/index.js",
"type": "module",
"scripts": {
"test": "./node_modules/.bin/mocha --timeout 30000",
"sync": "node ./bin/sync",
"example": "node ./example/example",
"testd": "UCI_ENV=dev ./node_modules/.bin/nodemon --exec 'npm run test'",
"testdd": "UCI_LOG_LEVEL='trace' npm run testd",
"testde": "UCI_LOG_LEVEL='warn' npm run testd",
"testl": "UCI_ENV=pro UCI_LOG_PATH=./test/test.log 0 npm run test || exit 0"
},
"author": "David Kebler",
"license": "MIT",
"engines": {
"node": ">=16.0.0"
},
"devDependencies": {
"chai": "^4.3.4",
"eslint": "^7.29.0",
"eslint-config-standard": "^16.0.3",
"mocha": "^9.x",
"nodemon": "^2.0.12",
"ondeath": "^1.0.0"
},
"keywords": [
"node.js",
"I2C",
"raspberryPi"
],
"name": "@uci-utils/sync",
"version": "0.0.10",
"description": "module to copy, maintain, and launch hardware modules on other machines",
"bin": {
"syncu": "./bin/sync-cli"
},
"repository": {
"type": "git",
"url": "git+https://github.com/uCOMmandIt/uci-remote-code.git"
},
"bugs": {
"url": "https://github.com/uCOMmandIt/uci-remote-code/issues"
},
"homepage": "https://github.com/uCOMmandIt/uci-remote-code#readme",
"dependencies": {
"@uci-utils/class-merge": "^1.1.0",
"@uci-utils/logger": "^0.1.0",
"@uci-utils/read-lines": "^0.3.0",
"@uci-utils/watcher": "^0.6.1",
"await-to-js": "^3.0.0",
"conf": "^10.0.1",
"debounce-fn": "^5.0.0",
"js-yaml": "^3.14.1",
"load-yaml-file": "^0.2.0",
"path-exists": "^4.0.0",
"write-yaml-file": "^4.2.0",
"yargs": "^17.0.1"
}
}

44
package.yml Executable file
View File

@ -0,0 +1,44 @@
name: '@uci-utils/sync'
version: 0.0.10
description: module to copy, maintain, and launch hardware modules on other machines
main: src/index.js
bin:
syncu: ./bin/sync-cli
type: module
scripts:
sync: node ./bin/sync
example: node ./example/example
test: ./node_modules/.bin/mocha --timeout 30000
testd: UCI_ENV=dev ./node_modules/.bin/nodemon --exec 'npm run test'
testdd: UCI_LOG_LEVEL='trace' npm run testd
testde: UCI_LOG_LEVEL='warn' npm run testd
testl: UCI_ENV=pro UCI_LOG_PATH=./test/test.log 0 npm run test || exit 0
author: David Kebler
license: MIT
repository:
type: git
url: git+https://github.com/uCOMmandIt/uci-remote-code.git
keywords:
- node.js
- I2C
- raspberryPi
bugs:
url: https://github.com/uCOMmandIt/uci-remote-code/issues
homepage: https://github.com/uCOMmandIt/uci-remote-code#readme
dependencies:
'@uci-utils/class-merge': ^1.1.0
'@uci-utils/logger': ^0.1.0
'@uci-utils/read-lines': ^0.3.0
'@uci-utils/watcher': ^0.6.1
await-to-js: ^3.0.0
conf: ^10.0.1
debounce-fn: ^5.0.0
load-yaml-file: ^0.2.0
path-exists: ^4.0.0
write-yaml-file: ^4.2.0
yargs: ^17.0.1
devDependencies:
chai: ^4.3.4
mocha: ^9.x
nodemon: ^2.0.12
ondeath: ^1.0.0

29
readme.md Normal file → Executable file
View File

@ -7,3 +7,32 @@
[![Dependencies](https://img.shields.io/david/uCOMmandIt/uci-pkg-template.svg)](https://david-dm.org/uCOMmandIt/uci-pkg-template)
[![devDependencies](https://img.shields.io/david/dev/uCOMmandIt/uci-pkg-template.svg)](https://david-dm.org/uCOMmandIt/uci-pkg-template?type=dev)
[![codecov](https://img.shields.io/codecov/c/github/uCOMmandIt/uci-pkg-template/master.svg)](https://codecov.io/gh/uCOMmandIt/uci-pkg-template)
# Why
# Class API
## public fields
syncHandler
jobsDir
sshDir
optionsDir
## getters
watching
watcher
getConfig
## setters
setConfig
setJobsDir
## public methods

4
src/index.js Normal file → Executable file
View File

@ -1,5 +1,5 @@
import Sync from './sync'
import Rsync from './rsync'
import Sync from './sync.js'
import Rsync from './rsync.js'
export { Sync, Rsync }
export default Sync

9
src/rsync.js Normal file → Executable file
View File

@ -1,5 +1,5 @@
var spawn = require('child_process').spawn
var path = require('path')
import { spawn } from'child_process'
import path from 'path'
/**
* Rsync is a wrapper class to configure and execute an `rsync` command
@ -881,8 +881,9 @@ exposeShortOption('specials')
*/
exposeShortOption('t', 'times')
// our awesome export product
module.exports = Rsync
export default Rsync
export { Rsync }
/* **** */

148
src/sync.js Normal file → Executable file
View File

@ -1,54 +1,79 @@
// local imports
import Rsync from './rsync'
import Rsync from './rsync.js'
// native imports
import { EventEmitter as Emitter } from 'events'
import { EventEmitter } from 'events'
import { dirname, normalize } from 'path'
// third party elements
import merge from '@uci-utils/class-merge'
import { readFile } from 'fs-read-data'
import loadYaml from 'load-yaml-file'
import Conf from 'conf'
import debounce from 'debounce-fn'
import pathExists from 'path-exists'
import to from 'await-to-js'
import { to } from 'await-to-js'
import yaml from 'js-yaml'
// uci imports
import logger from '@uci-utils/logger'
import Watcher from '@uci-utils/watcher'
// import Watcher from '../../watcher/src/watcher'
let log = {} // declare module wide log to be set during construction
class Sync extends merge(Rsync, Emitter) {
// class Sync extends merge(Rsync, Emitter) {
class Sync extends EventEmitter {
#config
// #debounce
#watcher
#cwd
#settings
#sources
#excludeFiles
#exclude
constructor(opts = {}) {
super()
log = logger({ package:'@uci/sync'})
this.opts = opts
this._debounce = opts.debounce || null
this.syncHandler = opts.syncHandler ? opts.syncHandler.bind(this) : defaultSyncHandler.bind(this)
this.#settings = Object.assign({},opts) // need deep clone?
// this.#debounce = opts.debounce || null
this.#settings.syncHandler = opts.syncHandler ? opts.syncHandler.bind(this) : defaultSyncHandler.bind(this)
// TODO if opts include source and destination then call loadJob with them
this.config = new Conf({projectName:'sync'})
this.jobsDir = process.env.SYNC_JOBS_DIR || opts.jobsDir || this.config.get('jobsDir') || dirname(this.config.path)
this.sshDir = opts.sshDir || this.config.get('sshDir') || `${this.jobsDir}/ssh`
this.optionsDir = opts.optionsDir || this.config.get('optionsDir') || `${this.jobsDir}/options`
console.log('settings',this.#settings)
this.#config = new Conf(
{
configName:this.#settings.configName,
cwd:this.#settings.configDir,
fileExtension: 'yaml',
serialize: yaml.safeDump,
deserialize: yaml.safeLoad
}
)
this.#config.set('test',"a test")
console.dir(this.#config.store)
this.#settings.jobsDir = process.env.SYNC_JOBS_DIR || this.#settings.jobsDir || this.#config.get('jobsDir') || dirname(this.#config.path)
this.#settings.sshDir = opts.sshDir || this.#config.get('sshDir') || `${this.jobsDir}/ssh`
this.#settings.optionsDir = this.#settings.optionsDir || this.#config.get('optionsDir') || `${this.jobsDir}/options`
log.debug({jobsDir:this.jobsDir, sshDir:this.sshDir, optionsDir:this.optionsDir, msg:'configuration file directories'})
this._watcher = new Watcher()
this.#watcher = new Watcher()
}
// getters and setters
get watching() {return this._watcher.watching } // watcher active?
get watcher() { return this._watcher} // in case one need set a listerner for other purposes
get defaultJobsDir() { return this.config.get('jobsDir', this.jobsDir) }
get watching() {return this.#watcher.watching } // watcher active?
get watcher() { return this.#watcher} // in case one need set a listerner for other purposes
get defaultJobsDir() { return this.#config.get('jobsDir', this.jobsDir) }
set defaultJobsDir(jobsDir) {
if (jobsDir==null) this.config.delete('jobsDir')
else this.config.set('jobsDir', jobsDir)
if (jobsDir==null) this.#config.delete('jobsDir')
else this.#config.set('jobsDir', jobsDir)
}
setConfig(name,val) { this.config.set(name,val)}
getConfig(name) { return this.config.get(name)}
setConfig(name,val) { this.#config.set(name,val)}
getConfig(name) { return this.#config.get(name)}
async setJobsDir(dir='',save) {
let res = await pathExists(dir)
if(res){
this.jobsDir=dir
if(save) this.setDefaultJobsDir()
this.#settings.jobsDir=dir
if(save) this.setDefaultJobsDir(dir)
return res
}
else {
@ -57,16 +82,23 @@ class Sync extends merge(Rsync, Emitter) {
}
}
// job and options processing
// internal private methods
async runJob(options) {
let res = await this.loadJob(options)
this.live()
await this.execute(this.opts)
async #readOptionsFile(filePath,type='job') {
let dir = {job:this.jobsDir,options:this.optionsDir,ssh:this.sshDir}
let [err,res] = await to(readFile(`${dir[type]}/${filePath}`))
if (err) {
[err,res] = await to(readFile(filePath))
if (err) {
err = {filePath:normalize(`${this.#cwd}/${dir[type]}/${filePath}`), error:err, type:type, msg:`unable to read ${filePath} options file`}
log.warn(err)
return err
}
}
return res
}
async loadJob (options) {
async #loadJob (options) {
if (typeof options ==='string') options = await this.readOptionsFile(options,'job')
if (!options.error) {
for (const option in options) {
@ -83,30 +115,26 @@ class Sync extends merge(Rsync, Emitter) {
}
} // end loop
this.dry() // dry run by default must .live() .unset('n')
let success = {options:options, cwd:this._cwd, command:this.command(), msg: 'job options processed sucessfully'}
let success = {options:options, cwd:this.#cwd, command:this.command(), msg: 'job options processed sucessfully'}
log.info(success)
return success
} throw options // options is error
}
async runJob(options) {
let res = await this.loadJob(options)
this.live()
await this.execute(this.opts)
return res
}
// async listJobFiles(dir) {
// return await getFiles(['**','*.yaml','*.yml'],{ onlyfiles:true, cwd:dir || this.jobsDir})
// }
async readOptionsFile(filePath,type='job') {
let dir = {job:this.jobsDir,options:this.optionsDir,ssh:this.sshDir}
let [err,res] = await to(readFile(`${dir[type]}/${filePath}`))
if (err) {
[err,res] = await to(readFile(filePath))
if (err) {
err = {filePath:normalize(`${this._cwd}/${dir[type]}/${filePath}`), error:err, type:type, msg:`unable to read ${filePath} options file`}
log.warn(err)
return err
}
}
return res
}
// executes a method on the instance (might be in prototype chain) which may take a value(s)
async processOption (method, value) {
@ -141,48 +169,48 @@ class Sync extends merge(Rsync, Emitter) {
async watch(cmd) {
// TODO make into switch ?
log.debug(`watch command ${cmd}`)
let opts = {source:this._sources, excludeFrom:this._excludeFiles, ignored:this._exclude }
let opts = {source:this.#sources, excludeFrom:this.#excludeFiles, ignored:this.#exclude }
if (isPlainObject(cmd) || cmd==null || (typeof cmd==='boolean' && cmd) || cmd==='init' || cmd==='add') {
if (cmd.wait) this.debounce(cmd)
if ( cmd.init || cmd==='init') await this._watcher.init(opts)
// if (cmd.wait) this.debounce(cmd)
if ( cmd.init || cmd==='init') await this.#watcher.init(opts)
return this
}
if (cmd==='remove') {
this._watcher.removeListener('changed', this.syncHandler )
this._watcher.remove()
this.#watcher.removeListener('changed', this.syncHandler )
this.#watcher.remove()
return
}
if (cmd ==='on'|| cmd==='start') {
this._watcher.on('changed', this.syncHandler)
this._watcher.start(opts)
this.#watcher.on('changed', this.syncHandler)
this.#watcher.start(opts)
return
}
if (cmd==='off' || cmd==='stop' ) {
this._watcher.removeListener('changed', this.syncHandler)
this._watcher.stop()
this.#watcher.removeListener('changed', this.syncHandler)
this.#watcher.stop()
return
}
if (cmd==='pause' ) {
this._watcher.removeListener('changed', this.syncHandler)
this.#watcher.removeListener('changed', this.syncHandler)
return
}
if (cmd==='resume') {
this._watcher.on('changed', this.syncHandler)
this.#watcher.on('changed', this.syncHandler)
return
}
}
debounce(opts) {
if (opts==null) this._debounce=null
this._debounce = opts
return this
}
// set debounce(opts) {
// if (opts==null) this.#debounce=null
// this.#debounce = opts
// return this
// }
sshu (file) {
if (file && (typeof options==='string')) {
@ -259,8 +287,8 @@ export default Sync
// default handler 'changed' event handler with optional debounce wrapper
function defaultSyncHandler(change={}) {
log.debug({file:change.file, type:change.type, msg:`file ${change.file} was ${change.type}`})
if (this._debounce==null) this.execute(this.opts)
else debounce(this.execute.bind(this),this._debounce)(this.opts)
// if (this.#debounce==null) this.execute(this.opts)
// else debounce(this.execute.bind(this),this.#debounce)(this.opts)
}
function isPlainObject (obj) {

13
test/files/jobs/local.yaml Executable file
View File

@ -0,0 +1,13 @@
source: ./test/files/source
destination: ./test/files/destination
flags: "av"
excludeFrom:
- ./test/files/source/.gitignore
- ./test/files/source/.npmignore
exclude:
- .gitignore
- .npmignore
set:
- delete
- delete-excluded
watch: true

View File

0
test/jobs/remote.yaml → test/files/jobs/remote.yaml Normal file → Executable file
View File

View File

0
test/source/.gitignore → test/files/source/.gitignore vendored Normal file → Executable file
View File

View File

0
test/source/.rcignore → test/files/source/.rcignore Normal file → Executable file
View File

View File

View File

@ -1,13 +0,0 @@
source: ./test/repo
destination: ./test/dest
flags: 'av'
excludeFrom:
- ./test/repo/.gitignore
- ./test/repo/.npmignore
exclude:
- .gitignore
- .npmignore
set:
- delete
- delete-excluded
watch:

33
test/sync.test.js Normal file → Executable file
View File

@ -1,20 +1,22 @@
import Sync from '../src/sync'
import Sync from '../src/sync.js'
import to from 'await-to-js'
import { expect } from 'chai'
import { it } from 'mocha'
import logger from '@uci-utils/logger'
// pause = require('@uci/utils').pPause
import { it, describe } from 'mocha'
describe('Sync Class Testing ',async ()=> {
let log = logger({})
before(async () => {
// log = logger({ package:'@uci/sync', id: 'sync-test' })
let sync = new Sync({jobsDir:'./test/jobs'})
await sync.loadJob('remote')
console.log('command to be run',sync.command())
sync.live()
sync.execute({cli:true})
before(async () => {
console.log('in before')
const opts = {
configDir:'./test/files',
configName:'sync_config'
}
let sync = new Sync(opts)
console.log(sync.getConfig)
// await sync.loadJob('remote')
// console.log('command to be run',sync.command())
// sync.live()
// sync.execute({cli:true})
// await sync.configure('./test/config/sync')
// log.info({cmd:sync.command(), msg:'Rsync Command that will Run'})
@ -22,6 +24,9 @@ describe('Sync Class Testing ',async ()=> {
// log.info('ready for testing')
})
describe('Sync Class Testing ',async ()=> {
// let log = logger({})
// after(async () => {
// remote.close()
// })