lloydzhou / mapred.io

mapreduce based on socket.io

mapred.io

mapreduce based on socket.io
You can use this module, use the browser to easily build mapreduce programming model based on parallel computing platform. On this platform, you can submit the page using javascript mapreduce tasks.

##install

npm install mapred.io

Server

var app = require('http').createServer(handler)
  , fs = require('fs')
  , io = require('mapred.io').listen(app)
  , parse = require('url').parse;
  
  app.listen(80);
  io.set('log level', 2);

function handler (req, res) {
  var info = parse(req.url, true);

  var hander = function (err, data) {
	if (err) {
	  res.writeHead(500);
	  return res.end('Error loading file');
	}

	res.writeHead(200);
	res.end(data);
  }
  if (info.pathname.match(/\/public/))
	fs.readFile(__dirname + info.pathname, hander )
  else if (info.pathname.match(/\/client/))
	fs.readFile(__dirname + '/client.html', hander )
  else fs.readFile(__dirname + '/job.html', hander )
}

Client

<script type="text/javascript" src="/socket.io/socket.io.js"></script>
<script type="text/javascript" src="/socket.io/mapred.io.js"></script>
<script type="text/javascript" >
var socket = io.connect();
var MapredClient = new MapredClient(socket);
</script>

Submit Job

<script type="text/javascript" src="/socket.io/socket.io.js"></script>
<script type="text/javascript" src="/socket.io/mapred.io.js"></script>
<script type="text/javascript" >

var input = [
	['frase primera', 'primer trozo de informacion para procesado primer trozo'],
	['segunda frase', 'segundo trozo de informacion trozo de'],
	['cacho 3', 'otro trozo para ser procesado otro otro otro trozo'],
	['cuarta frase', 'primer trozo de informacion para procesado primer trozo'],
	['frase 5', 'segundo trozo de informacion trozo de']
], inputs = input, i;
//for ( i = 0 ; i &lt; 20000 ; i ++ ) inputs = inputs.concat(input);

socket.emit('job', { map: (function(key, value){
    var list = [], aux = {};
        value = value.split(' ');
        value.forEach(function(w){
            aux[w] = (aux[w] || 0) + 1;
        });
        for(var k in aux){
            list.push([k, aux[k]]);
        }
        return list;
    }).toString(), reduce: (function(key, values){
        var sum = 0;
        values.forEach(function(e){
            sum += e;
        });
        return sum;
    }).toString(), inputs:inputs)

change log

2013-11-29 update mapred.io-client.js can be using in browser and command line, add client.js can run as a node.

To do

1. Performance Test
2. Client program (the standalone client not on the browser).
3. Stored the datas (the server do not have enough memory to handle large data, Need a fast persistent storage framework. Just using the store of socket.io default is "MemoryStore", you can using "RedisStore")

update the screeshot