lc3t35 / discophusion Goto Github PK

# launch_test bash
root@discotest:/# ./test.sh
/... cut env results.../
/usr/local/lib/python2.7/dist-packages/disco/fileutils.py:141: UserWarning: Duplicate name: 'usr/local/lib/python2.7/dist-packages/disco/job.py'
  self.write(file, file)
/usr/local/lib/python2.7/dist-packages/disco/fileutils.py:141: UserWarning: Duplicate name: 'usr/local/lib/python2.7/dist-packages/disco/worker/classic/worker.py'
  self.write(file, file)
Job@581:550f0:7708e:
Status: [map] 0 pending, 0 waiting, 1 running, 0 done, 0 failed
2014/08/29 21:40:00 master     New job initialized!
2014/08/29 21:40:00 master     Created job "Job@581:550f0:7708e"
2014/08/29 21:40:00 master     Stage map scheduled with 1 tasks
2014/08/29 21:40:00 master     map:0 assigned to localhost
2014/08/29 21:40:05 localhost  MSG: [map:0] Done: 9732 entries mapped
2014/08/29 21:40:05 localhost  MSG: [map:0] Results sent to master
2014/08/29 21:40:05 localhost  DONE: [map:0] Task finished in 0:00:05.208
2014/08/29 21:40:06 master     SYS: [map:0] Received results from localhost
2014/08/29 21:40:06 master     Stage map finished in 0:00:06.337
2014/08/29 21:40:06 master     Stage map_shuffle scheduled with 1 tasks
2014/08/29 21:40:06 master     map_shuffle:1 assigned to localhost
2014/08/29 21:40:07 localhost  MSG: [map_shuffle:1] Shuffled 1 inputs into 1 label(s)
2014/08/29 21:40:07 localhost  MSG: [map_shuffle:1] Results sent to master
2014/08/29 21:40:07 localhost  DONE: [map_shuffle:1] Task finished in 0:00:01.021
2014/08/29 21:40:09 master     SYS: [map_shuffle:1] Received results from localhost
2014/08/29 21:40:09 master     Stage map_shuffle finished in 0:00:02.502
2014/08/29 21:40:09 master     Stage reduce scheduled with 1 tasks
2014/08/29 21:40:09 master     reduce:2 assigned to localhost
Status: [reduce] 0 pending, 0 waiting, 1 running, 0 done, 0 failed
2014/08/29 21:40:10 localhost  MSG: [reduce:2] Done: 70143 entries reduced
2014/08/29 21:40:10 localhost  MSG: [reduce:2] Results sent to master
2014/08/29 21:40:10 localhost  DONE: [reduce:2] Task finished in 0:00:01.566
2014/08/29 21:40:12 master     SYS: [reduce:2] Received results from localhost
2014/08/29 21:40:12 master     Stage reduce finished in 0:00:02.890
2014/08/29 21:40:12 master      READY: Job done in 0:00:11.736
 Status: [reduce] 0 pending, 0 waiting, 0 running, 1 done, 0 failed
2014/08/29 21:40:00 master     New job initialized!
2014/08/29 21:40:00 master     Created job "Job@581:550f0:7708e"
2014/08/29 21:40:00 master     Stage map scheduled with 1 tasks
2014/08/29 21:40:00 master     map:0 assigned to localhost
2014/08/29 21:40:05 localhost  MSG: [map:0] Done: 9732 entries mapped
2014/08/29 21:40:05 localhost  MSG: [map:0] Results sent to master
2014/08/29 21:40:05 localhost  DONE: [map:0] Task finished in 0:00:05.208
2014/08/29 21:40:06 master     SYS: [map:0] Received results from localhost
2014/08/29 21:40:06 master     Stage map finished in 0:00:06.337
2014/08/29 21:40:06 master     Stage map_shuffle scheduled with 1 tasks
2014/08/29 21:40:06 master     map_shuffle:1 assigned to localhost
2014/08/29 21:40:07 localhost  MSG: [map_shuffle:1] Shuffled 1 inputs into 1 label(s)
2014/08/29 21:40:07 localhost  MSG: [map_shuffle:1] Results sent to master
2014/08/29 21:40:07 localhost  DONE: [map_shuffle:1] Task finished in 0:00:01.021
2014/08/29 21:40:09 master     SYS: [map_shuffle:1] Received results from localhost
2014/08/29 21:40:09 master     Stage map_shuffle finished in 0:00:02.502
2014/08/29 21:40:09 master     Stage reduce scheduled with 1 tasks
2014/08/29 21:40:09 master     reduce:2 assigned to localhost
2014/08/29 21:40:10 localhost  MSG: [reduce:2] Done: 70143 entries reduced
2014/08/29 21:40:10 localhost  MSG: [reduce:2] Results sent to master
2014/08/29 21:40:10 localhost  DONE: [reduce:2] Task finished in 0:00:01.566
2014/08/29 21:40:12 master     SYS: [reduce:2] Received results from localhost
2014/08/29 21:40:12 master     Stage reduce finished in 0:00:02.890
2014/08/29 21:40:12 master      READY: Job done in 0:00:11.736
(and much later)
 Traceback (most recent call last):
  File "/disco/examples/util/count_words.py", line 16, in <module>
    for word, count in result_iterator(job.wait(show=True)):
  File "/usr/local/lib/python2.7/dist-packages/disco/core.py", line 406, in classic_iterator
    for input in util.inputlist(urls, settings=settings):
  File "/usr/local/lib/python2.7/dist-packages/disco/util.py", line 283, in inputlist
    for input in inputs) if inp]
  File "/usr/local/lib/python2.7/dist-packages/disco/util.py", line 42, in chainify
    return list(chain(*iterable))
  File "/usr/local/lib/python2.7/dist-packages/disco/util.py", line 283, in <genexpr>
    for input in inputs) if inp]
  File "/usr/local/lib/python2.7/dist-packages/disco/util.py", line 273, in inputexpand
    return zip(*(parse_dir(i, label=label) for i in iterify(input)))
  File "/usr/local/lib/python2.7/dist-packages/disco/util.py", line 273, in <genexpr>
    return zip(*(parse_dir(i, label=label) for i in iterify(input)))
  File "/usr/local/lib/python2.7/dist-packages/disco/util.py", line 243, in parse_dir
    return [url for lab, url, size in sorted(read_index(dir)) if label in (None, lab)]
  File "/usr/local/lib/python2.7/dist-packages/disco/util.py", line 258, in read_index
    file = open_url(proxy_url(dir, to_master=False))
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 104, in open_url
    return open_remote(url, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 110, in open_remote
    return Connection(urlresolve(url), token)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 146, in __init__
    self.read(1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 174, in read
    bytes = self._read_chunk(size if size > 0 else CHUNK_SIZE)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 194, in _read_chunk
    headers=headers)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 71, in request
    return request(method, url, data=data, headers=headers, sleep=sleep + 1)
  File "/usr/local/lib/python2.7/dist-packages/disco/comm.py", line 69, in request
    raise CommError(errmsg, url, status)
disco.error.CommError: Unable to access resource (http://localhost:8989/disco/localhost/4d/Job@581:550f0:7708e/.disco/reduce-2-1409348410903037.results): [Errno 111] Connection refused

(disco_8989_master@discomaster)1> Eshell V5.10.4  (abort with ^G)
(disco_8989_master@discomaster)1> 21:15:39.005 [info] Application lager started on node disco_8989_ma
ster@discomaster
(disco_8989_master@discomaster)1> 21:15:39.102 [info] Application inets started on node disco_8989_ma
ster@discomaster
(disco_8989_master@discomaster)1> 21:15:39.103 [info] DISCO BOOTS
(disco_8989_master@discomaster)1> 21:15:39.106 [info] Disco proxy enabled
(disco_8989_master@discomaster)1> 21:15:39.112 [info] DDFS master starts
(disco_8989_master@discomaster)1> 21:15:39.122 [info] Event server starts
(disco_8989_master@discomaster)1> 21:15:39.130 [info] Disco config starts
(disco_8989_master@discomaster)1> 21:15:39.134 [info] DISCO SERVER STARTS
(disco_8989_master@discomaster)1> 21:15:39.136 [info] Fair scheduler starts
(disco_8989_master@discomaster)1> 21:15:39.136 [info] Scheduler uses fair policy
(disco_8989_master@discomaster)1> 21:15:39.138 [info] Fair scheduler: Fair policy
(disco_8989_master@discomaster)1> 21:15:39.143 [info] Config table updated
(disco_8989_master@discomaster)1> 21:15:39.834 [info] Starting node "disco_8989_slave" on "localhost"
 ("localhost")
(disco_8989_master@discomaster)1> 21:15:39.841 [info] web server (mochiweb) starts
(disco_8989_master@discomaster)1> 21:15:39.841 [info] Application disco started on node disco_8989_ma
ster@discomaster
(disco_8989_master@discomaster)1> 21:15:40.434 [info] initialized job "Job@581:54b3c:65a73" with pipe
line [{<<"map">>,split,false},{<<"map_shuffle">>,group_node,false},{<<"reduce">>,group_all,false}] an
d inputs [{0,{data,{0,0,[{<<"http://discoproject.org/media/text/chekhov.txt">>,"discoproject.org"}]}}
}]
(disco_8989_master@discomaster)1> 21:15:42.032 [info] ddfs_node initialized on disco_8989_master@disc
omaster with volumes: ["vol0"]
(disco_8989_master@discomaster)1> 21:15:42.048 [info] ddfs_node starts on disco_8989_master@discomast
er
(disco_8989_master@discomaster)1> 21:15:42.048 [info] lock_server starts on disco_8989_slave@localhos
t
(disco_8989_master@discomaster)1> 21:15:42.048 [info] Node started at disco_8989_slave@localhost (rep
orting as disco_8989_master@discomaster) on "localhost"
(disco_8989_master@discomaster)1> 21:15:42.053 [info] ddfs_node initialized on disco_8989_slave@local
host with volumes: ["vol0"]
(disco_8989_master@discomaster)1> 21:15:42.082 [info] Tempgc: error listing "/usr/local/var/disco/dat
a/localhost": {error,enoent}
(disco_8989_master@discomaster)1> 21:15:42.082 [info] Tempgc: one pass completed on disco_8989_slave@
localhost
(disco_8989_master@discomaster)1> 21:15:42.119 [info] Started ddfs_put at disco_8989_slave@localhost 
on port 8990
(disco_8989_master@discomaster)1> 21:15:42.125 [info] ddfs_node starts on disco_8989_slave@localhost
(disco_8989_master@discomaster)1> 21:15:42.380 [info] Node started at disco_8989_slave@localhost (rep
orting as disco_8989_slave@localhost) on "localhost"
(disco_8989_master@discomaster)1> 21:15:45.289 [info] Starting proxy at pid "106"
(disco_8989_master@discomaster)1> 21:15:53.225 [info] Job Job@581:54b3c:65a73 done, results: [[<<"dir
://localhost/disco/localhost/c2/Job@581:54b3c:65a73/.disco/reduce-2-1409346951560645.results">>]]
(disco_8989_master@discomaster)1> 21:16:28.921 [info] Config table updated
(disco_8989_master@discomaster)1> 21:16:28.934 [info] Killing pid "106"
(disco_8989_master@discomaster)1> 21:16:36.378 [info] Starting proxy at pid "195"
(disco_8989_master@discomaster)1> 21:16:40.979 [info] Config table updated
(disco_8989_master@discomaster)1> 21:16:40.981 [info] Killing pid "195"
(disco_8989_master@discomaster)1> 21:16:48.039 [info] Starting proxy at pid "207"
(disco_8989_master@discomaster)1> 21:20:39.161 [info] GC: initializing
(disco_8989_master@discomaster)1> 21:20:39.208 [info] GC: average disk utilization: 0, over utilized 
nodes: 0, under utilized nodes: 0
(disco_8989_master@discomaster)1> 21:20:39.208 [info] GC: entering gc phase
(disco_8989_master@discomaster)1> 21:20:39.219 [info] GC: found 0 blob, 0 tag candidates on disco_898
9_slave@localhost
(disco_8989_master@discomaster)1> 21:20:39.219 [info] GC: disk information for disco_8989_slave@local
host (free: 30532520 bytes, used: 8505860 bytes)
(disco_8989_master@discomaster)1> 21:20:39.233 [info] Node GC Stats for disco_8989_slave@localhost: {
tag,"kept",{0,0},"deleted",{0,0}}  {blob,"kept",{0,0},"deleted",{0,0}}
(disco_8989_master@discomaster)1> 21:20:39.233 [info] GC: Pruning +deleted
(disco_8989_master@discomaster)1> 21:20:39.233 [info] Total GC Stats: {tag,"kept",{0,0},"deleted",{0,
0}}  {blob,"kept",{0,0},"deleted",{0,0}}
(disco_8989_master@discomaster)1> 21:20:39.234 [info] GC: average disk utilization: 0.217884553611087
36, over utilized nodes: 0, under utilized nodes: 0
(disco_8989_master@discomaster)1> 21:20:39.234 [info] GC: average disk utilization: 0.217884553611087
36, over utilized nodes: 0, under utilized nodes: 0
(disco_8989_master@discomaster)1> 21:20:39.234 [info] GC: entering rr_blobs phase
(disco_8989_master@discomaster)1> 21:20:39.234 [info] GC: sent 0 blob replication requests, entering 
rr_blobs_wait
(disco_8989_master@discomaster)1> 21:20:39.234 [info] GC: replication ending with Ref 0, TO 0
(disco_8989_master@discomaster)1> 21:20:39.234 [info] GC: done with blob replication, replicating tag
s (0 pending)
(disco_8989_master@discomaster)1> 21:20:39.234 [info] GC: 0 tags updated/replication done, done with 
GC!
(disco_8989_master@discomaster)1> 21:20:39.235 [info] GC terminated.
(disco_8989_master@discomaster)1> 21:40:00.500 [info] initialized job "Job@581:550f0:7708e" with pipe
line [{<<"map">>,split,false},{<<"map_shuffle">>,group_node,false},{<<"reduce">>,group_all,false}] an
d inputs [{0,{data,{0,0,[{<<"http://discoproject.org/media/text/chekhov.txt">>,"discoproject.org"}]}}
}]
(disco_8989_master@discomaster)1> 21:40:12.233 [info] Job Job@581:550f0:7708e done, results: [[<<"dir
://localhost/disco/localhost/4d/Job@581:550f0:7708e/.disco/reduce-2-1409348410903037.results">>]]
(disco_8989_master@discomaster)1> 21:40:12.233 [info] Job has already finished: {gen_server,call,[<0.
740.0>,{schedule_local,["localhost"]},infinity]}
(disco_8989_master@discomaster)1>