From 9382497c60d95b905971958b96c94ef7b8c04388 Mon Sep 17 00:00:00 2001 From: Harish K Date: Wed, 24 Sep 2014 11:42:45 +0530 Subject: [PATCH 1/7] create README.md README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..ade32a2 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +node-job-manager +================ + +A flexible Node.js asynchronous task runner class + From bee25f17f27c5a1e26847de343039091c043097f Mon Sep 17 00:00:00 2001 From: "Harish.K" Date: Wed, 24 Sep 2014 12:37:59 +0530 Subject: [PATCH 2/7] fix: update state function updatestate should not change the order of workers in the tmpPool remove code repeatation fix: actual concurrent tasks running was concurrency-1. fixed by placing cb after returnTOPool at $doWork_ functions --- src/JobManager.js | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/JobManager.js b/src/JobManager.js index f7c7651..91ad45c 100644 --- a/src/JobManager.js +++ b/src/JobManager.js @@ -10,7 +10,7 @@ function JobManager(opts){ this.notifyAt = opts.notifyAt|| 3; this.concurrency = opts.concurrency || 20; this.minConcurrency = opts.minConcurrency != null ? opts.minConcurrency : 2; - + // state this.state = STATE.NOT_RUNNING; @@ -22,6 +22,8 @@ function JobManager(opts){ this.isLoadingTakingPlace = false; this.endReached = false; this.runningTasks = 0; + this.tmpPoolLen=0; + this.tmpPool = []; } /* * JobManager.prototype.__defineSetter__( 'workers', function(workers){ @@ -31,26 +33,24 @@ function JobManager(opts){ */ JobManager.prototype.updateState = function(){ - var workers = this.workers; - var tmpPoolBlock = workers.length && Math.ceil( this.concurrency/workers.length ) , i; - var tmpPool = []; - for( i=0; i < tmpPoolBlock; i++){ - tmpPool = tmpPool.concat( this.workers ); + this.updateTmpPool( this.concurrency ); +}; + +JobManager.prototype.updateTmpPool = function(newVal){ + if( newVal > this.tmpPoolLen ){ + var tmpPool = this.tmpPool; + var additionalPoolBlocks = Math.ceil( ( newVal - this.tmpPoolLen )/this.workers.length ), i; + for( i=0; i < additionalPoolBlocks; i++){ + tmpPool = tmpPool.concat( this.workers ); + } + this.tmpPoolLen += additionalPoolBlocks * this.workers.length ; + this.tmpPool = tmpPool; } - this.tmpPool = tmpPool; - this.tmpPoolLen = tmpPoolBlock * this.workers.length ; }; JobManager.prototype.setConcurrency = function( newVal ){ if( newVal > this.minConcurrency ){ - if( newVal > this.tmpPoolLen ){ - var tmpPool = this.tmpPool; - var additionalPoolBlocks = Math.ceil( ( newVal - this.tmpPoolLen )/this.workers.length ), i; - for( i=0; i < additionalPoolBlocks; i++){ - tmpPool = tmpPool.concat( this.workers ); - } - this.tmpPoolLen += additionalPoolBlocks * this.workers.length ; - } + this.updateTmpPool(newVal); this.concurrency = newVal; } }; @@ -90,11 +90,11 @@ JobManager.prototype.$doWork_ = function( cb ){ self.$onLoadMore(); } } - cb(); if( self.state == STATE.NOT_RUNNING ){ if( ( self.runningTasks == 1 ) && self.onStopped ) { self.onStopped(); } } self.returnToPool( worker ); + cb(); }); }; From bdccf6ccc37aeb845dd1e81756b4cd493fb5205d Mon Sep 17 00:00:00 2001 From: "Harish.K" Date: Wed, 24 Sep 2014 13:15:19 +0530 Subject: [PATCH 3/7] Readme --- README.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ade32a2..7aca92e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,30 @@ node-job-manager ================ -A flexible Node.js asynchronous task runner class +JobManager is a multipurpose, flexible, asynchronous task runner class written in Node.js. +Overview +-------- +### A JobManager instance need the following informations to get started. they are +* ```workers```: An array of worker instances that are used to do the work. +* ```onLoadMore```: A function that tells how to retrieve tasks that need to processed with workers. +* ```work```: a function that does the real task. It is called with following syntax: worker( task, worker, cb ). task and worker are are belongs to the available tasks and workers. cb is the callback function that should be called after work is done. +* ```onError```: A function that is called when a Error is occurred during 'work' function. it is called with following syntax: onError(err, task, worker); +* ```onStopped```: A function that is called when all tasks are processed. + + +### JobManager instance is controlled by the following configuration options +* ```concurrency```: An integer that tells how many workers should be run a time. Note that, concurrency cab be higher than no.of workers. In this case, one worker may be used to process more than one task at a time. +* ```minConcurrency```: An integer. concurrency can not be decreased below this value. +* ```notifyAt```: if ( no.tasks/concurrency < notifyAt ), onLoadMore function is called to fetch more tasks. +* ```endReached```: A Boolean. if it is set to true, onLoadMore will not be called further. and thus, execution of JobManager will stop after processing all the currently available tasks. +* ```setConcurrency```: A function that is used to change concurrency at runtime. + +### Following variables reveals current state of JobManager instance. +* ```state```: NOT_RUNNING | RUNNING. +* ```isLoadingTakingPlace```: onLoadMore function is taking place. + + +Demo +---- + A pure javascript demo at http://harish2704.github.io/jobmanager-demo/test.html will give a more cleat idea. This demo uses the same JobManager class to do the animation. From 56463869fcc378a90e7ad6c2ecab61ed2282f30b Mon Sep 17 00:00:00 2001 From: "Harish.K" Date: Wed, 24 Sep 2014 13:34:32 +0530 Subject: [PATCH 4/7] Update readme and package.json --- README.md | 20 ++++++++++++++++++++ package.json | 6 +----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7aca92e..e619050 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,27 @@ Overview * ```state```: NOT_RUNNING | RUNNING. * ```isLoadingTakingPlace```: onLoadMore function is taking place. +Usage +----- +```javascript +var JobManager = require('job-manager').JobManager; +var jm = new JobManager({ configuration options}); +jm.tasks = tasks; +jm.onLoadMore = function(cb){// code; if( !tasks ){ this.endReached = true; } cb()} +jm.work = function(task, worker,cb){ // worker.process( task, cb ); } +jm.onError = function( err, task, worker){ log( err, task, worker ); } +jm.onStopped = function(){ // log('Finished'); cb(); } +jm.start(); +// jm.stop(); +``` Demo ---- A pure javascript demo at http://harish2704.github.io/jobmanager-demo/test.html will give a more cleat idea. This demo uses the same JobManager class to do the animation. + + +Self promotion +-------------- +* I am a javascript freelancer. You can hire me. +* star my repos. + diff --git a/package.json b/package.json index 293fe66..3d14c62 100644 --- a/package.json +++ b/package.json @@ -16,11 +16,7 @@ "email": "harish2704@gmail.com" }, "license": "BSD-2-Clause", - "readme": "ERROR: No README data found!", + "readme": "README.md", "_id": "job-manager@0.0.1", - "dist": { - "shasum": "06c8c7558143c473f748db83876de9d0a449c764" - }, - "_resolved": "git+https://github.com/harish2704/node-job-manager#971f97e5f5985d58614fd3dbcca583dba250e964", "_from": "git+https://github.com/harish2704/node-job-manager" } From 0d8c5b7b8838bdd7cf4c5f03aaec075ffe81798d Mon Sep 17 00:00:00 2001 From: "Harish.K" Date: Wed, 24 Sep 2014 13:41:55 +0530 Subject: [PATCH 5/7] Update package.json --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 3d14c62..b8b602c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "job-manager", - "version": "0.0.1", + "version": "0.0.2", "description": "A task manager class wich runs tasks with given workers, running N tasks at a time, automatically load works, pause resume etc", "main": "src/JobManager.js", "scripts": { @@ -16,7 +16,6 @@ "email": "harish2704@gmail.com" }, "license": "BSD-2-Clause", - "readme": "README.md", "_id": "job-manager@0.0.1", "_from": "git+https://github.com/harish2704/node-job-manager" } From 0a56774bb032f074b5b8b167eaca2f0dcbc98849 Mon Sep 17 00:00:00 2001 From: "Harish.K" Date: Wed, 24 Sep 2014 13:53:06 +0530 Subject: [PATCH 6/7] update package.json --- package.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index b8b602c..e36daec 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,15 @@ { "name": "job-manager", - "version": "0.0.2", + "version": "0.0.3", "description": "A task manager class wich runs tasks with given workers, running N tasks at a time, automatically load works, pause resume etc", "main": "src/JobManager.js", "scripts": { "test": "mocha -R spec test/*.spec.js" }, + "repository": { + "type": "git", + "url": "https://github.com/harish2704/node-job-manager" + }, "keywords": [ "job-manager", "asynchronous", @@ -16,6 +20,7 @@ "email": "harish2704@gmail.com" }, "license": "BSD-2-Clause", - "_id": "job-manager@0.0.1", - "_from": "git+https://github.com/harish2704/node-job-manager" + "bugs": { + "url": "https://github.com/harish2704/node-job-manager/issues" + } } From 8a5a4fb3d50865accc557febdf3c389da9b5837b Mon Sep 17 00:00:00 2001 From: "Harish.K" Date: Thu, 25 Sep 2014 22:31:11 +0530 Subject: [PATCH 7/7] Update package.json and readme --- README.md | 8 ++++---- package.json | 12 +++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e619050..fe35606 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ node-job-manager ================ -JobManager is a multipurpose, flexible, asynchronous task runner class written in Node.js. +JobManager is a multipurpose, flexible, asynchronous queue manager class written in Nodejs. Queue is processed using given workers. It is possible to change concurrency and load bulk tasks at run time. See demo for better understanding Overview -------- -### A JobManager instance need the following informations to get started. they are +### A JobManager instance need the following informations to get started. They are, * ```workers```: An array of worker instances that are used to do the work. * ```onLoadMore```: A function that tells how to retrieve tasks that need to processed with workers. * ```work```: a function that does the real task. It is called with following syntax: worker( task, worker, cb ). task and worker are are belongs to the available tasks and workers. cb is the callback function that should be called after work is done. @@ -29,10 +29,10 @@ Usage ```javascript var JobManager = require('job-manager').JobManager; var jm = new JobManager({ configuration options}); -jm.tasks = tasks; +jm.workers = workers; jm.onLoadMore = function(cb){// code; if( !tasks ){ this.endReached = true; } cb()} jm.work = function(task, worker,cb){ // worker.process( task, cb ); } -jm.onError = function( err, task, worker){ log( err, task, worker ); } +jm.onError = function( err, task, worker){ // log( err, task, worker ); } jm.onStopped = function(){ // log('Finished'); cb(); } jm.start(); // jm.stop(); diff --git a/package.json b/package.json index e36daec..84f674f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "job-manager", - "version": "0.0.3", - "description": "A task manager class wich runs tasks with given workers, running N tasks at a time, automatically load works, pause resume etc", + "version": "0.0.4", + "description": "A queue manager class which process tasks with given workers, with adjustable concurrency. It can automatically load tasks, pause resume etc", "main": "src/JobManager.js", "scripts": { "test": "mocha -R spec test/*.spec.js" @@ -13,7 +13,13 @@ "keywords": [ "job-manager", "asynchronous", - "parallel" + "parallel", + "queue", + "async", + "job", + "task", + "concurrency", + "concurrent" ], "author": { "name": "Harish.K",