-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathcase397_4.log
1052 lines (1052 loc) · 216 KB
/
case397_4.log
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
+ filename=case397
+ requestdistribution=
+ RESULT_DIR=/home/hadoop/git/physical_design/caseresult/case397
+ TEN=10
+ SIXTY=60
+ SLEEP_TIME=60
+ YCSB_DIR=/home/hadoop/ycsb-0.1.4
+ number_of_operations=25000000
+ number_of_threads=20
+ number_of_records=25000000
+ output_file_suffix=.dat
+ experiment_list_file=/home/hadoop/git/physical_design/case_dir/case397.list
+ clean=false
+ create=false
+ load=false
+ random_read=false
+ workloada=false
+ workloadb=false
+ workloadc=false
+ workloadc1=false
+ scan=false
+ all=false
+ for i in '$@'
+ '[' case397 == all ']'
+ '[' case397 == clean ']'
+ '[' case397 == create ']'
+ '[' case397 == load ']'
+ '[' case397 == random_read ']'
+ '[' case397 == scan ']'
+ '[' case397 == workloada ']'
+ '[' case397 == workloadb ']'
+ '[' case397 == workloadc ']'
+ '[' case397 == workloadc1 ']'
+ '[' case397 == zipfian ']'
+ '[' case397 == uniform ']'
+ for i in '$@'
+ '[' workloadc1 == all ']'
+ '[' workloadc1 == clean ']'
+ '[' workloadc1 == create ']'
+ '[' workloadc1 == load ']'
+ '[' workloadc1 == random_read ']'
+ '[' workloadc1 == scan ']'
+ '[' workloadc1 == workloada ']'
+ '[' workloadc1 == workloadb ']'
+ '[' workloadc1 == workloadc ']'
+ '[' workloadc1 == workloadc1 ']'
+ workloadc1=true
+ for i in '$@'
+ '[' zipfian == all ']'
+ '[' zipfian == clean ']'
+ '[' zipfian == create ']'
+ '[' zipfian == load ']'
+ '[' zipfian == random_read ']'
+ '[' zipfian == scan ']'
+ '[' zipfian == workloada ']'
+ '[' zipfian == workloadb ']'
+ '[' zipfian == workloadc ']'
+ '[' zipfian == workloadc1 ']'
+ '[' zipfian == zipfian ']'
+ requestdistribution=zipfian
+ let counter=1
+ cd /home/hadoop/ycsb-0.1.4
++ pwd
+ echo /home/hadoop/ycsb-0.1.4
/home/hadoop/ycsb-0.1.4
+ read line
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ false
+ true
+ echo workloadc1...
workloadc1...
+ echo 'start time: '
start time:
++ date +%s
+ echo 1407999628
1407999628
++ echo /home/hadoop/git/physical_design/caseresult/case397/workloadc1_1.dat
+ ./workloadc1 25000000 20 /home/hadoop/git/physical_design/caseresult/case397/workloadc1_1.dat
Loading workload...
Starting test.
2014-08-14 00:00:28:642 0 sec: 0 operations;
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:host.name=sceplus-vm43.almaden.ibm.com
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:java.version=1.6.0_31
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:java.vendor=Sun Microsystems Inc.
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:java.home=/usr/lib/jvm/java-6-openjdk-amd64/jre
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:java.class.path=/home/hadoop/ycsb-0.1.4/hbase-binding/lib/hbase-binding-0.1.4.jar:/home/hadoop/ycsb-0.1.4/hbase-binding/conf:/home/hadoop/ycsb-0.1.4/core/lib/core-0.1.4.jar:/home/hadoop/ycsb-0.1.4/jdbc-binding/conf
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:java.library.path=/usr/lib/jvm/java-6-openjdk-amd64/jre/lib/amd64/server:/usr/lib/jvm/java-6-openjdk-amd64/jre/lib/amd64:/usr/lib/jvm/java-6-openjdk-amd64/jre/../lib/amd64:/usr/java/packages/lib/amd64:/usr/lib/jni:/lib:/usr/lib
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:java.compiler=<NA>
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:os.name=Linux
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:os.arch=amd64
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:os.version=3.13.0-24-generic
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:user.name=hadoop
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:user.home=/home/hadoop
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Client environment:user.dir=/home/hadoop/ycsb-0.1.4
14/08/14 00:00:29 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=sceplus-vm49.almaden.ibm.com:2181,sceplus-vm48.almaden.ibm.com:2181 sessionTimeout=90000 watcher=hconnection-0x689d16c2, quorum=sceplus-vm49.almaden.ibm.com:2181,sceplus-vm48.almaden.ibm.com:2181, baseZNode=/hbase
14/08/14 00:00:29 INFO zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x689d16c2 connecting to ZooKeeper ensemble=sceplus-vm49.almaden.ibm.com:2181,sceplus-vm48.almaden.ibm.com:2181
14/08/14 00:00:29 INFO zookeeper.ClientCnxn: Opening socket connection to server master/9.1.143.58:2181. Will not attempt to authenticate using SASL (unknown error)
14/08/14 00:00:29 INFO zookeeper.ClientCnxn: Socket connection established to master/9.1.143.58:2181, initiating session
14/08/14 00:00:29 INFO zookeeper.ClientCnxn: Session establishment complete on server master/9.1.143.58:2181, sessionid = 0x47d43fd8de0007, negotiated timeout = 90000
14/08/14 00:00:29 WARN util.DynamicClassLoader: Failed to identify the fs of dir hdfs://master:54310/hbase/lib, ignored
org.apache.hadoop.ipc.RemoteException: Server IPC version 9 cannot communicate with client version 4
at org.apache.hadoop.ipc.Client.call(Client.java:1113)
at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229)
at com.sun.proxy.$Proxy5.getProtocolVersion(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:622)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:85)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:62)
at com.sun.proxy.$Proxy5.getProtocolVersion(Unknown Source)
at org.apache.hadoop.ipc.RPC.checkVersion(RPC.java:422)
at org.apache.hadoop.hdfs.DFSClient.createNamenode(DFSClient.java:183)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:281)
at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:245)
at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:100)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:1446)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:67)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:1464)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:263)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:187)
at org.apache.hadoop.hbase.util.DynamicClassLoader.<init>(DynamicClassLoader.java:104)
at org.apache.hadoop.hbase.protobuf.ProtobufUtil.<clinit>(ProtobufUtil.java:202)
at org.apache.hadoop.hbase.ClusterId.parseFrom(ClusterId.java:64)
at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:69)
at org.apache.hadoop.hbase.client.ZooKeeperRegistry.getClusterId(ZooKeeperRegistry.java:83)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.retrieveClusterId(HConnectionManager.java:858)
at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.<init>(HConnectionManager.java:663)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:534)
at org.apache.hadoop.hbase.client.HConnectionManager.createConnection(HConnectionManager.java:415)
at org.apache.hadoop.hbase.client.HConnectionManager.createConnection(HConnectionManager.java:394)
at org.apache.hadoop.hbase.client.HConnectionManager.getConnection(HConnectionManager.java:275)
at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:197)
at org.apache.hadoop.hbase.client.HTable.<init>(HTable.java:159)
at com.yahoo.ycsb.db.HBaseClient.getHTable(HBaseClient.java:118)
at com.yahoo.ycsb.db.HBaseClient.update(HBaseClient.java:302)
at com.yahoo.ycsb.db.HBaseClient.insert(HBaseClient.java:357)
at com.yahoo.ycsb.DBWrapper.insert(DBWrapper.java:148)
at com.yahoo.ycsb.workloads.CoreWorkload.doTransactionInsert(CoreWorkload.java:639)
at com.yahoo.ycsb.workloads.CoreWorkload.doTransaction(CoreWorkload.java:487)
at com.yahoo.ycsb.ClientThread.run(Client.java:241)
2014-08-14 00:00:38:643 10 sec: 190252 operations; 19023.3 current ops/sec; [INSERT AverageLatency(us)=201.36] [READ AverageLatency(us)=34817.98]
2014-08-14 00:00:48:644 20 sec: 213390 operations; 2313.57 current ops/sec; [INSERT AverageLatency(us)=466.92] [READ AverageLatency(us)=28952.43]
2014-08-14 00:00:59:185 30 sec: 230833 operations; 1654.78 current ops/sec; [INSERT AverageLatency(us)=7016.21] [READ AverageLatency(us)=18048.5]
2014-08-14 00:01:09:186 40 sec: 344595 operations; 11375.06 current ops/sec; [INSERT AverageLatency(us)=4077.01] [READ AverageLatency(us)=37337.25]
2014-08-14 00:01:19:186 50 sec: 435336 operations; 9074.1 current ops/sec; [INSERT AverageLatency(us)=1694.81] [READ AverageLatency(us)=45953.92]
2014-08-14 00:01:29:187 60 sec: 471907 operations; 3656.73 current ops/sec; [INSERT AverageLatency(us)=5828.71] [READ AverageLatency(us)=25025.38]
2014-08-14 00:01:39:187 70 sec: 576572 operations; 10466.5 current ops/sec; [INSERT AverageLatency(us)=850.78] [READ AverageLatency(us)=41037.99]
2014-08-14 00:01:49:188 80 sec: 652586 operations; 7600.64 current ops/sec; [INSERT AverageLatency(us)=2862.45] [READ AverageLatency(us)=35653.49]
2014-08-14 00:01:59:189 90 sec: 737909 operations; 8531.45 current ops/sec; [INSERT AverageLatency(us)=2059.83] [READ AverageLatency(us)=34268.74]
2014-08-14 00:02:09:189 100 sec: 821519 operations; 8361 current ops/sec; [INSERT AverageLatency(us)=2351.95] [READ AverageLatency(us)=33667.09]
2014-08-14 00:02:19:190 110 sec: 910479 operations; 8895.11 current ops/sec; [INSERT AverageLatency(us)=1063.87] [READ AverageLatency(us)=32091.88]
2014-08-14 00:02:29:191 120 sec: 991103 operations; 8061.59 current ops/sec; [INSERT AverageLatency(us)=2056.4] [READ AverageLatency(us)=29622.65]
2014-08-14 00:02:39:192 130 sec: 1090688 operations; 9957.5 current ops/sec; [INSERT AverageLatency(us)=2090.97] [READ AverageLatency(us)=30100.7]
2014-08-14 00:02:49:193 140 sec: 1144603 operations; 5390.96 current ops/sec; [INSERT AverageLatency(us)=4490.92] [READ AverageLatency(us)=28984.63]
2014-08-14 00:02:59:195 150 sec: 1247206 operations; 10258.25 current ops/sec; [INSERT AverageLatency(us)=1027.6] [READ AverageLatency(us)=36365.97]
2014-08-14 00:03:09:196 160 sec: 1323074 operations; 7586.04 current ops/sec; [INSERT AverageLatency(us)=1925.19] [READ AverageLatency(us)=26361.45]
2014-08-14 00:03:19:197 170 sec: 1422684 operations; 9960 current ops/sec; [INSERT AverageLatency(us)=2280.91] [READ AverageLatency(us)=40625.63]
2014-08-14 00:03:29:199 180 sec: 1522264 operations; 9956.01 current ops/sec; [INSERT AverageLatency(us)=1800.88] [READ AverageLatency(us)=29797.12]
2014-08-14 00:03:39:200 190 sec: 1588631 operations; 6636.04 current ops/sec; [INSERT AverageLatency(us)=1840.04] [READ AverageLatency(us)=33012.84]
2014-08-14 00:03:49:201 200 sec: 1688279 operations; 9963.8 current ops/sec; [INSERT AverageLatency(us)=2108.98] [READ AverageLatency(us)=36020.14]
2014-08-14 00:03:59:202 210 sec: 1696544 operations; 826.42 current ops/sec; [INSERT AverageLatency(us)=6394.14] [READ AverageLatency(us)=30479.95]
2014-08-14 00:04:09:203 220 sec: 1696544 operations; 0 current ops/sec;
2014-08-14 00:04:19:204 230 sec: 1811571 operations; 11501.55 current ops/sec; [INSERT AverageLatency(us)=3648.66] [READ AverageLatency(us)=151749.66]
2014-08-14 00:04:29:206 240 sec: 1855545 operations; 4396.52 current ops/sec; [INSERT AverageLatency(us)=3136.83] [READ AverageLatency(us)=19662.59]
2014-08-14 00:04:39:207 250 sec: 1949100 operations; 9354.56 current ops/sec; [INSERT AverageLatency(us)=1348] [READ AverageLatency(us)=36512]
2014-08-14 00:04:49:209 260 sec: 2048777 operations; 9966.7 current ops/sec; [INSERT AverageLatency(us)=1778.89] [READ AverageLatency(us)=36429.28]
2014-08-14 00:04:59:210 270 sec: 2143123 operations; 9432.71 current ops/sec; [INSERT AverageLatency(us)=2312.77] [READ AverageLatency(us)=33429.08]
2014-08-14 00:05:09:211 280 sec: 2219552 operations; 7642.14 current ops/sec; [INSERT AverageLatency(us)=1890.36] [READ AverageLatency(us)=26553.58]
2014-08-14 00:05:19:213 290 sec: 2285917 operations; 6635.17 current ops/sec; [INSERT AverageLatency(us)=1845.41] [READ AverageLatency(us)=26454.23]
2014-08-14 00:05:29:214 300 sec: 2385555 operations; 9962.8 current ops/sec; [INSERT AverageLatency(us)=2048.18] [READ AverageLatency(us)=135281.78]
2014-08-14 00:05:39:215 310 sec: 2423493 operations; 3793.42 current ops/sec; [INSERT AverageLatency(us)=1491.86] [READ AverageLatency(us)=23110.22]
2014-08-14 00:05:49:217 320 sec: 2527879 operations; 10436.51 current ops/sec; [INSERT AverageLatency(us)=2107.23] [READ AverageLatency(us)=45781.38]
2014-08-14 00:05:59:218 330 sec: 2618041 operations; 9015.3 current ops/sec; [INSERT AverageLatency(us)=1960.27] [READ AverageLatency(us)=38309.32]
2014-08-14 00:06:09:219 340 sec: 2717658 operations; 9960.7 current ops/sec; [INSERT AverageLatency(us)=2049.01] [READ AverageLatency(us)=36300.2]
2014-08-14 00:06:19:220 350 sec: 2724499 operations; 684.03 current ops/sec; [INSERT AverageLatency(us)=7568.06] [READ AverageLatency(us)=29141.54]
2014-08-14 00:06:29:222 360 sec: 2724499 operations; 0 current ops/sec;
2014-08-14 00:06:39:223 370 sec: 2724499 operations; 0 current ops/sec;
2014-08-14 00:06:49:224 380 sec: 2724499 operations; 0 current ops/sec;
2014-08-14 00:06:59:225 390 sec: 2724499 operations; 0 current ops/sec;
2014-08-14 00:07:09:225 400 sec: 2724499 operations; 0 current ops/sec;
2014-08-14 00:07:19:226 410 sec: 2724499 operations; 0 current ops/sec;
2014-08-14 00:07:29:227 420 sec: 2738629 operations; 1412.86 current ops/sec; [INSERT AverageLatency(us)=13.51] [READ AverageLatency(us)=2775267.47]
2014-08-14 00:07:39:229 430 sec: 2841843 operations; 10319.34 current ops/sec; [INSERT AverageLatency(us)=12060.3] [READ AverageLatency(us)=91909.03]
2014-08-14 00:07:49:230 440 sec: 2850662 operations; 881.81 current ops/sec; [INSERT AverageLatency(us)=750.47] [READ AverageLatency(us)=22049.97]
2014-08-14 00:07:59:231 450 sec: 2850662 operations; 0 current ops/sec;
2014-08-14 00:08:09:232 460 sec: 2850662 operations; 0 current ops/sec;
2014-08-14 00:08:19:233 470 sec: 2850662 operations; 0 current ops/sec;
2014-08-14 00:08:29:234 480 sec: 2911525 operations; 6085.69 current ops/sec; [INSERT AverageLatency(us)=13113.85] [READ AverageLatency(us)=381257.89]
2014-08-14 00:08:39:235 490 sec: 3030810 operations; 11927.31 current ops/sec; [INSERT AverageLatency(us)=191.89] [READ AverageLatency(us)=36051.03]
2014-08-14 00:08:49:237 500 sec: 3030810 operations; 0 current ops/sec;
2014-08-14 00:08:59:238 510 sec: 3030810 operations; 0 current ops/sec;
2014-08-14 00:09:09:239 520 sec: 3030810 operations; 0 current ops/sec;
2014-08-14 00:09:19:240 530 sec: 3175905 operations; 14509.5 current ops/sec; [INSERT AverageLatency(us)=5722.44] [READ AverageLatency(us)=44901]
2014-08-14 00:09:29:241 540 sec: 3297049 operations; 12111.98 current ops/sec; [INSERT AverageLatency(us)=912.92] [READ AverageLatency(us)=42546.38]
2014-08-14 00:09:39:242 550 sec: 3405641 operations; 10858.11 current ops/sec; [INSERT AverageLatency(us)=1166.72] [READ AverageLatency(us)=45714.34]
2014-08-14 00:09:49:243 560 sec: 3495823 operations; 9017.3 current ops/sec; [INSERT AverageLatency(us)=2057.94] [READ AverageLatency(us)=35687.25]
2014-08-14 00:09:59:244 570 sec: 3556320 operations; 6049.1 current ops/sec; [INSERT AverageLatency(us)=2979.69] [READ AverageLatency(us)=39171.64]
2014-08-14 00:10:09:246 580 sec: 3634861 operations; 7852.53 current ops/sec; [INSERT AverageLatency(us)=2141.04] [READ AverageLatency(us)=46164.55]
2014-08-14 00:10:19:247 590 sec: 3719297 operations; 8442.76 current ops/sec; [INSERT AverageLatency(us)=2082.69] [READ AverageLatency(us)=56661.99]
2014-08-14 00:10:29:249 600 sec: 3773291 operations; 5398.32 current ops/sec; [INSERT AverageLatency(us)=4005.51] [READ AverageLatency(us)=25680.04]
2014-08-14 00:10:39:250 610 sec: 3870792 operations; 9749.13 current ops/sec; [INSERT AverageLatency(us)=709.89] [READ AverageLatency(us)=41848.94]
2014-08-14 00:10:49:251 620 sec: 3965495 operations; 9469.35 current ops/sec; [INSERT AverageLatency(us)=2020.27] [READ AverageLatency(us)=47325.89]
2014-08-14 00:10:59:253 630 sec: 3994756 operations; 2925.51 current ops/sec; [INSERT AverageLatency(us)=4223.25] [READ AverageLatency(us)=29831.09]
2014-08-14 00:11:09:254 640 sec: 3994756 operations; 0 current ops/sec;
2014-08-14 00:11:19:255 650 sec: 3994756 operations; 0 current ops/sec;
2014-08-14 00:11:29:313 660 sec: 4076348 operations; 8112.15 current ops/sec; [INSERT AverageLatency(us)=6472.6] [READ AverageLatency(us)=218165.01]
2014-08-14 00:11:39:443 670 sec: 4133092 operations; 5601.58 current ops/sec; [INSERT AverageLatency(us)=3541.34] [READ AverageLatency(us)=18067.79]
2014-08-14 00:11:49:444 680 sec: 4240543 operations; 10744.03 current ops/sec; [INSERT AverageLatency(us)=628.8] [READ AverageLatency(us)=51698.1]
2014-08-14 00:11:59:445 690 sec: 4340152 operations; 9959.9 current ops/sec; [INSERT AverageLatency(us)=2207.03] [READ AverageLatency(us)=52896.64]
2014-08-14 00:12:09:446 700 sec: 4435034 operations; 9487.25 current ops/sec; [INSERT AverageLatency(us)=1654.01] [READ AverageLatency(us)=34078.35]
2014-08-14 00:12:19:447 710 sec: 4534285 operations; 9924.11 current ops/sec; [INSERT AverageLatency(us)=1984.94] [READ AverageLatency(us)=60792.3]
2014-08-14 00:12:29:448 720 sec: 4582100 operations; 4781.02 current ops/sec; [INSERT AverageLatency(us)=1322.77] [READ AverageLatency(us)=23215.83]
2014-08-14 00:12:39:449 730 sec: 4681675 operations; 9956.5 current ops/sec; [INSERT AverageLatency(us)=2228.85] [READ AverageLatency(us)=50415.64]
2014-08-14 00:12:49:450 740 sec: 4732573 operations; 5089.29 current ops/sec; [INSERT AverageLatency(us)=2973.89] [READ AverageLatency(us)=33289.06]
2014-08-14 00:12:59:454 750 sec: 4827240 operations; 9462.91 current ops/sec; [INSERT AverageLatency(us)=2316.31] [READ AverageLatency(us)=48765.01]
2014-08-14 00:13:09:455 760 sec: 4865605 operations; 3836.12 current ops/sec; [INSERT AverageLatency(us)=5567.83] [READ AverageLatency(us)=39350.12]
2014-08-14 00:13:19:456 770 sec: 4975988 operations; 11037.2 current ops/sec; [INSERT AverageLatency(us)=257.37] [READ AverageLatency(us)=52490.44]
2014-08-14 00:13:29:458 780 sec: 4975988 operations; 0 current ops/sec;
2014-08-14 00:13:39:459 790 sec: 4985180 operations; 919.02 current ops/sec; [INSERT AverageLatency(us)=5535.43] [READ AverageLatency(us)=220456.07]
2014-08-14 00:13:49:460 800 sec: 5088182 operations; 10299.17 current ops/sec; [INSERT AverageLatency(us)=5281.93] [READ AverageLatency(us)=63218.2]
2014-08-14 00:13:59:461 810 sec: 5125673 operations; 3748.73 current ops/sec; [INSERT AverageLatency(us)=5981.9] [READ AverageLatency(us)=28267.67]
2014-08-14 00:14:09:462 820 sec: 5236767 operations; 11108.29 current ops/sec; [INSERT AverageLatency(us)=178.36] [READ AverageLatency(us)=64511.07]
2014-08-14 00:14:19:464 830 sec: 5331628 operations; 9484.2 current ops/sec; [INSERT AverageLatency(us)=2035.43] [READ AverageLatency(us)=48676.72]
2014-08-14 00:14:29:465 840 sec: 5419827 operations; 8819.02 current ops/sec; [INSERT AverageLatency(us)=2015.78] [READ AverageLatency(us)=47764.8]
2014-08-14 00:14:39:466 850 sec: 5426963 operations; 713.53 current ops/sec; [INSERT AverageLatency(us)=1271.1] [READ AverageLatency(us)=16225.45]
2014-08-14 00:14:49:467 860 sec: 5475369 operations; 4840.12 current ops/sec; [INSERT AverageLatency(us)=5259.19] [READ AverageLatency(us)=67304.44]
2014-08-14 00:14:59:469 870 sec: 5549370 operations; 7398.62 current ops/sec; [INSERT AverageLatency(us)=3587.76] [READ AverageLatency(us)=38599.21]
2014-08-14 00:15:09:470 880 sec: 5627235 operations; 7785.72 current ops/sec; [INSERT AverageLatency(us)=2546.01] [READ AverageLatency(us)=48451.87]
2014-08-14 00:15:19:471 890 sec: 5707663 operations; 8042 current ops/sec; [INSERT AverageLatency(us)=1502.11] [READ AverageLatency(us)=25814.92]
2014-08-14 00:15:29:472 900 sec: 5791678 operations; 8400.66 current ops/sec; [INSERT AverageLatency(us)=1834.9] [READ AverageLatency(us)=38844.64]
2014-08-14 00:15:39:473 910 sec: 5862777 operations; 7109.19 current ops/sec; [INSERT AverageLatency(us)=2154.87] [READ AverageLatency(us)=41588.64]
2014-08-14 00:15:49:475 920 sec: 5957640 operations; 9485.35 current ops/sec; [INSERT AverageLatency(us)=2117.24] [READ AverageLatency(us)=39110.04]
2014-08-14 00:15:59:476 930 sec: 6057224 operations; 9957.4 current ops/sec; [INSERT AverageLatency(us)=1985.42] [READ AverageLatency(us)=42913.77]
2014-08-14 00:16:09:477 940 sec: 6104888 operations; 4765.45 current ops/sec; [INSERT AverageLatency(us)=2516.66] [READ AverageLatency(us)=21123.53]
2014-08-14 00:16:19:478 950 sec: 6186295 operations; 8139.89 current ops/sec; [INSERT AverageLatency(us)=1901.75] [READ AverageLatency(us)=83384.52]
2014-08-14 00:16:29:479 960 sec: 6186295 operations; 0 current ops/sec;
2014-08-14 00:16:39:480 970 sec: 6186295 operations; 0 current ops/sec;
2014-08-14 00:16:49:481 980 sec: 6186295 operations; 0 current ops/sec;
2014-08-14 00:16:59:482 990 sec: 6186295 operations; 0 current ops/sec;
2014-08-14 00:17:09:484 1000 sec: 6186295 operations; 0 current ops/sec;
2014-08-14 00:17:19:485 1010 sec: 6186295 operations; 0 current ops/sec;
2014-08-14 00:17:29:485 1020 sec: 6186295 operations; 0 current ops/sec;
2014-08-14 00:17:39:486 1030 sec: 6186295 operations; 0 current ops/sec;
14/08/14 00:17:47 INFO client.AsyncProcess: #14, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10051 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #12, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014, retrying after 10066 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #8, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10010 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #17, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10035 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #13, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10071 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #6, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014, retrying after 10001 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #18, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10064 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #4, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014, retrying after 10006 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #20, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10060 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #3, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014, retrying after 10056 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #2, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:05 PDT 2014, retrying after 10074 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #11, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:15 PDT 2014, retrying after 10003 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #10, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10007 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #5, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10044 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #7, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10093 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #16, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10003 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #9, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014, retrying after 10020 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #1, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014, retrying after 10000 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #19, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014, retrying after 10051 ms, replay 4696 ops.
14/08/14 00:17:47 INFO client.AsyncProcess: #15, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36574 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:15 PDT 2014, retrying after 10066 ms, replay 4696 ops.
2014-08-14 00:17:49:486 1040 sec: 6186295 operations; 0 current ops/sec;
2014-08-14 00:17:59:487 1050 sec: 6192848 operations; 655.23 current ops/sec; [INSERT AverageLatency(us)=10.8] [READ AverageLatency(us)=3638881.77]
14/08/14 00:18:01 INFO client.AsyncProcess: #8, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:02 INFO client.AsyncProcess: #12, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014
14/08/14 00:18:02 INFO client.AsyncProcess: #14, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:05 INFO client.AsyncProcess: #7, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:06 INFO client.AsyncProcess: #19, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:06 INFO client.AsyncProcess: #6, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014
14/08/14 00:18:06 INFO client.AsyncProcess: #15, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:15 PDT 2014
14/08/14 00:18:06 INFO client.AsyncProcess: #10, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:06 INFO client.AsyncProcess: #20, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:06 INFO client.AsyncProcess: #17, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:06 INFO client.AsyncProcess: #11, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:15 PDT 2014
2014-08-14 00:18:09:488 1060 sec: 6246985 operations; 5413.16 current ops/sec; [INSERT AverageLatency(us)=18925.16] [READ AverageLatency(us)=25755.78]
14/08/14 00:18:10 INFO client.AsyncProcess: #3, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014
14/08/14 00:18:10 INFO client.AsyncProcess: #13, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:10 INFO client.AsyncProcess: #2, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:05 PDT 2014
14/08/14 00:18:10 INFO client.AsyncProcess: #4, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014
14/08/14 00:18:10 INFO client.AsyncProcess: #16, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:10 INFO client.AsyncProcess: #1, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014
14/08/14 00:18:10 INFO client.AsyncProcess: #5, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:10 INFO client.AsyncProcess: #18, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:09 PDT 2014
14/08/14 00:18:10 INFO client.AsyncProcess: #9, table=usertable, attempt=11/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:16:10 PDT 2014
2014-08-14 00:18:19:489 1070 sec: 6316067 operations; 6907.51 current ops/sec; [INSERT AverageLatency(us)=16692.61] [READ AverageLatency(us)=76661.08]
2014-08-14 00:18:29:490 1080 sec: 6450963 operations; 13488.25 current ops/sec; [INSERT AverageLatency(us)=782.47] [READ AverageLatency(us)=80088.82]
2014-08-14 00:18:39:491 1090 sec: 6495026 operations; 4405.86 current ops/sec; [INSERT AverageLatency(us)=3949.14] [READ AverageLatency(us)=47256.9]
2014-08-14 00:18:49:493 1100 sec: 6495026 operations; 0 current ops/sec;
2014-08-14 00:18:59:587 1110 sec: 6535144 operations; 3974.05 current ops/sec; [INSERT AverageLatency(us)=11.4] [READ AverageLatency(us)=719096.49]
2014-08-14 00:19:09:591 1120 sec: 6660951 operations; 12576.93 current ops/sec; [INSERT AverageLatency(us)=2084.14] [READ AverageLatency(us)=56186.55]
2014-08-14 00:19:19:592 1130 sec: 6694891 operations; 3393.32 current ops/sec; [INSERT AverageLatency(us)=6586.76] [READ AverageLatency(us)=25310.84]
2014-08-14 00:19:29:593 1140 sec: 6812552 operations; 11764.92 current ops/sec; [INSERT AverageLatency(us)=496.34] [READ AverageLatency(us)=39426.26]
2014-08-14 00:19:39:594 1150 sec: 6906404 operations; 9384.26 current ops/sec; [INSERT AverageLatency(us)=2206.59] [READ AverageLatency(us)=74366.93]
2014-08-14 00:19:49:595 1160 sec: 6949114 operations; 4270.57 current ops/sec; [INSERT AverageLatency(us)=1544.1] [READ AverageLatency(us)=27628.03]
2014-08-14 00:19:59:596 1170 sec: 7060849 operations; 11172.38 current ops/sec; [INSERT AverageLatency(us)=2111.64] [READ AverageLatency(us)=63867.96]
2014-08-14 00:20:09:597 1180 sec: 7156614 operations; 9575.54 current ops/sec; [INSERT AverageLatency(us)=2040.8] [READ AverageLatency(us)=38129.31]
2014-08-14 00:20:19:599 1190 sec: 7204766 operations; 4814.24 current ops/sec; [INSERT AverageLatency(us)=2605.01] [READ AverageLatency(us)=46234.29]
2014-08-14 00:20:29:601 1200 sec: 7233808 operations; 2903.62 current ops/sec; [INSERT AverageLatency(us)=9.44] [READ AverageLatency(us)=12148.6]
2014-08-14 00:20:39:602 1210 sec: 7233808 operations; 0 current ops/sec;
2014-08-14 00:20:49:603 1220 sec: 7301359 operations; 6754.42 current ops/sec; [INSERT AverageLatency(us)=8135.63] [READ AverageLatency(us)=46699.93]
2014-08-14 00:20:59:604 1230 sec: 7380823 operations; 7945.61 current ops/sec; [INSERT AverageLatency(us)=1893.58] [READ AverageLatency(us)=42552.15]
2014-08-14 00:21:09:605 1240 sec: 7485173 operations; 10433.96 current ops/sec; [INSERT AverageLatency(us)=1832.7] [READ AverageLatency(us)=39548.55]
2014-08-14 00:21:19:607 1250 sec: 7580025 operations; 9483.3 current ops/sec; [INSERT AverageLatency(us)=1920.9] [READ AverageLatency(us)=45702.18]
2014-08-14 00:21:29:608 1260 sec: 7651174 operations; 7114.19 current ops/sec; [INSERT AverageLatency(us)=2040.82] [READ AverageLatency(us)=30172.46]
2014-08-14 00:21:39:609 1270 sec: 7750805 operations; 9962.1 current ops/sec; [INSERT AverageLatency(us)=1929.52] [READ AverageLatency(us)=44772.76]
2014-08-14 00:21:49:610 1280 sec: 7775475 operations; 2466.75 current ops/sec; [INSERT AverageLatency(us)=9169.2] [READ AverageLatency(us)=181635.27]
2014-08-14 00:21:59:611 1290 sec: 7894495 operations; 11900.81 current ops/sec; [INSERT AverageLatency(us)=545.5] [READ AverageLatency(us)=25608.75]
2014-08-14 00:22:09:612 1300 sec: 7993253 operations; 9874.81 current ops/sec; [INSERT AverageLatency(us)=2172.51] [READ AverageLatency(us)=41545.25]
2014-08-14 00:22:19:613 1310 sec: 8047060 operations; 5380.16 current ops/sec; [INSERT AverageLatency(us)=4031.82] [READ AverageLatency(us)=31494.97]
2014-08-14 00:22:29:615 1320 sec: 8163444 operations; 11636.07 current ops/sec; [INSERT AverageLatency(us)=593.95] [READ AverageLatency(us)=34161.87]
2014-08-14 00:22:39:616 1330 sec: 8165438 operations; 199.38 current ops/sec; [INSERT AverageLatency(us)=88333.35] [READ AverageLatency(us)=27878.17]
2014-08-14 00:22:49:617 1340 sec: 8165438 operations; 0 current ops/sec;
2014-08-14 00:22:59:618 1350 sec: 8279149 operations; 11369.96 current ops/sec; [INSERT AverageLatency(us)=1751.4] [READ AverageLatency(us)=280310.92]
2014-08-14 00:23:09:619 1360 sec: 8471802 operations; 19263.37 current ops/sec; [INSERT AverageLatency(us)=266.31] [READ AverageLatency(us)=58524.97]
2014-08-14 00:23:19:760 1371 sec: 8531899 operations; 5926.14 current ops/sec; [INSERT AverageLatency(us)=2641.56] [READ AverageLatency(us)=33005.79]
2014-08-14 00:23:29:761 1381 sec: 8531899 operations; 0 current ops/sec;
2014-08-14 00:23:39:785 1391 sec: 8595399 operations; 6334.8 current ops/sec; [INSERT AverageLatency(us)=4327.28] [READ AverageLatency(us)=260215.57]
2014-08-14 00:23:49:788 1401 sec: 8718580 operations; 12314.41 current ops/sec; [INSERT AverageLatency(us)=888.42] [READ AverageLatency(us)=27886.03]
2014-08-14 00:23:59:789 1411 sec: 8813776 operations; 9518.65 current ops/sec; [INSERT AverageLatency(us)=1983.75] [READ AverageLatency(us)=38967.86]
2014-08-14 00:24:09:790 1421 sec: 8889945 operations; 7616.14 current ops/sec; [INSERT AverageLatency(us)=2510.42] [READ AverageLatency(us)=43209.05]
2014-08-14 00:24:19:791 1431 sec: 8997656 operations; 10770.02 current ops/sec; [INSERT AverageLatency(us)=1369.17] [READ AverageLatency(us)=31932.89]
2014-08-14 00:24:29:792 1441 sec: 9056981 operations; 5931.91 current ops/sec; [INSERT AverageLatency(us)=2816.23] [READ AverageLatency(us)=34470.09]
2014-08-14 00:24:39:793 1451 sec: 9127507 operations; 7051.89 current ops/sec; [INSERT AverageLatency(us)=2524.44] [READ AverageLatency(us)=50053.29]
2014-08-14 00:24:49:794 1461 sec: 9199792 operations; 7227.78 current ops/sec; [INSERT AverageLatency(us)=2657.33] [READ AverageLatency(us)=26123.05]
2014-08-14 00:24:59:795 1471 sec: 9289190 operations; 8938.91 current ops/sec; [INSERT AverageLatency(us)=1131.38] [READ AverageLatency(us)=37914.57]
2014-08-14 00:25:09:796 1481 sec: 9373245 operations; 8404.66 current ops/sec; [INSERT AverageLatency(us)=2003.82] [READ AverageLatency(us)=37672.55]
2014-08-14 00:25:19:798 1491 sec: 9472807 operations; 9954.21 current ops/sec; [INSERT AverageLatency(us)=2079.3] [READ AverageLatency(us)=35150.65]
2014-08-14 00:25:29:798 1501 sec: 9514786 operations; 4197.9 current ops/sec; [INSERT AverageLatency(us)=4521.81] [READ AverageLatency(us)=35381.17]
2014-08-14 00:25:39:799 1511 sec: 9514786 operations; 0 current ops/sec;
2014-08-14 00:25:49:921 1521 sec: 9564936 operations; 4954.55 current ops/sec; [INSERT AverageLatency(us)=1759.44] [READ AverageLatency(us)=388477.66]
2014-08-14 00:25:59:922 1531 sec: 9690983 operations; 12603.44 current ops/sec; [INSERT AverageLatency(us)=1815.57] [READ AverageLatency(us)=62414.25]
2014-08-14 00:26:09:923 1541 sec: 9795080 operations; 10408.66 current ops/sec; [INSERT AverageLatency(us)=1817.7] [READ AverageLatency(us)=37375.76]
2014-08-14 00:26:19:924 1551 sec: 9885907 operations; 9081.79 current ops/sec; [INSERT AverageLatency(us)=2238.22] [READ AverageLatency(us)=48683.68]
2014-08-14 00:26:29:925 1561 sec: 9928183 operations; 4227.18 current ops/sec; [INSERT AverageLatency(us)=1364.44] [READ AverageLatency(us)=59390.06]
2014-08-14 00:26:39:926 1571 sec: 9928183 operations; 0 current ops/sec;
2014-08-14 00:26:49:927 1581 sec: 9956644 operations; 2845.82 current ops/sec; [INSERT AverageLatency(us)=16788.82] [READ AverageLatency(us)=232776.21]
2014-08-14 00:26:59:930 1591 sec: 10120495 operations; 16380.19 current ops/sec; [INSERT AverageLatency(us)=530.75] [READ AverageLatency(us)=49859.68]
2014-08-14 00:27:09:931 1601 sec: 10236834 operations; 11632.74 current ops/sec; [INSERT AverageLatency(us)=413.49] [READ AverageLatency(us)=71122.18]
2014-08-14 00:27:19:933 1611 sec: 10334227 operations; 9737.35 current ops/sec; [INSERT AverageLatency(us)=1977.91] [READ AverageLatency(us)=44539.95]
2014-08-14 00:27:29:934 1621 sec: 10377193 operations; 4296.17 current ops/sec; [INSERT AverageLatency(us)=5134.29] [READ AverageLatency(us)=46733.19]
2014-08-14 00:27:39:935 1631 sec: 10485710 operations; 10850.61 current ops/sec; [INSERT AverageLatency(us)=702.88] [READ AverageLatency(us)=54040.3]
2014-08-14 00:27:49:936 1641 sec: 10596938 operations; 11121.69 current ops/sec; [INSERT AverageLatency(us)=1649.82] [READ AverageLatency(us)=65678.44]
2014-08-14 00:27:59:937 1651 sec: 10663371 operations; 6642.64 current ops/sec; [INSERT AverageLatency(us)=1482.15] [READ AverageLatency(us)=31573.55]
2014-08-14 00:28:09:939 1661 sec: 10666497 operations; 312.54 current ops/sec; [INSERT AverageLatency(us)=34511.05] [READ AverageLatency(us)=202854.53]
2014-08-14 00:28:19:940 1671 sec: 10704665 operations; 3816.42 current ops/sec; [INSERT AverageLatency(us)=33.02] [READ AverageLatency(us)=418410.28]
2014-08-14 00:28:29:941 1681 sec: 10806945 operations; 10226.98 current ops/sec; [INSERT AverageLatency(us)=3545.75] [READ AverageLatency(us)=61365.88]
2014-08-14 00:28:39:943 1691 sec: 10896979 operations; 9001.6 current ops/sec; [INSERT AverageLatency(us)=574.98] [READ AverageLatency(us)=77562.95]
2014-08-14 00:28:49:944 1701 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:28:59:945 1711 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:29:09:946 1721 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:29:19:947 1731 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:29:29:948 1741 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:29:39:949 1751 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:29:49:950 1761 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:29:59:951 1771 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:30:09:952 1781 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:30:19:953 1791 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:30:29:954 1801 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:30:39:955 1811 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:30:49:956 1821 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:30:59:957 1831 sec: 10896979 operations; 0 current ops/sec;
14/08/14 00:31:04 INFO client.AsyncProcess: #7, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10070 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #6, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10034 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #1, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10068 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #5, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10062 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #14, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 10033 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #18, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:29 PDT 2014, retrying after 10005 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #3, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10049 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #2, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 10042 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #9, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10074 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #19, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 10080 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #10, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 10034 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #11, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10071 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #16, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10008 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #17, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 10006 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #20, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 10029 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #12, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:29 PDT 2014, retrying after 10026 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #4, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10032 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #15, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10004 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #8, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 10029 ms, replay 4696 ops.
14/08/14 00:31:04 INFO client.AsyncProcess: #13, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10009 ms, replay 4696 ops.
2014-08-14 00:31:09:958 1841 sec: 10896979 operations; 0 current ops/sec;
14/08/14 00:31:14 INFO client.AsyncProcess: #6, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10004 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #1, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10067 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #18, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:29 PDT 2014, retrying after 10019 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #7, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10020 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #14, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 10058 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #2, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 10066 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #16, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10099 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #10, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 10047 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #5, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10094 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #3, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10073 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #17, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 10088 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #11, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10082 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #19, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 10011 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #20, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 10009 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #9, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10082 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #15, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10004 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #12, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:29 PDT 2014, retrying after 10006 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #13, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 10056 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #4, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 10040 ms, replay 4696 ops.
14/08/14 00:31:14 INFO client.AsyncProcess: #8, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 10048 ms, replay 4696 ops.
2014-08-14 00:31:19:959 1851 sec: 10896979 operations; 0 current ops/sec;
14/08/14 00:31:24 INFO client.AsyncProcess: #6, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 20154 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #18, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:29 PDT 2014, retrying after 20166 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #7, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 20154 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #1, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 20014 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #14, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 20055 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #10, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 20093 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #15, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 20077 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #20, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 20121 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #19, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014, retrying after 20109 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #2, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 20145 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #12, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:29 PDT 2014, retrying after 20068 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #3, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 20002 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #16, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 20005 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #17, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 20144 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #5, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 20027 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #4, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 20016 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #13, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014, retrying after 20025 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #11, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 20027 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #9, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014, retrying after 20196 ms, replay 4696 ops.
14/08/14 00:31:24 INFO client.AsyncProcess: #8, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36578 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014, retrying after 20033 ms, replay 4696 ops.
2014-08-14 00:31:29:959 1861 sec: 10896979 operations; 0 current ops/sec;
2014-08-14 00:31:39:960 1871 sec: 10896979 operations; 0 current ops/sec;
14/08/14 00:31:45 INFO client.AsyncProcess: #1, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014
14/08/14 00:31:45 INFO client.AsyncProcess: #14, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014
14/08/14 00:31:45 INFO client.AsyncProcess: #16, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014
14/08/14 00:31:45 INFO client.AsyncProcess: #6, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014
14/08/14 00:31:45 INFO client.AsyncProcess: #3, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014
14/08/14 00:31:46 INFO client.AsyncProcess: #9, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014
14/08/14 00:31:48 INFO client.AsyncProcess: #17, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014
2014-08-14 00:31:49:961 1881 sec: 10969096 operations; 7210.98 current ops/sec; [INSERT AverageLatency(us)=5468.21] [READ AverageLatency(us)=3718977.13]
14/08/14 00:31:53 INFO client.AsyncProcess: #2, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014
14/08/14 00:31:53 INFO client.AsyncProcess: #8, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:30 PDT 2014
14/08/14 00:31:53 INFO client.AsyncProcess: #19, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014
14/08/14 00:31:53 INFO client.AsyncProcess: #18, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:29 PDT 2014
14/08/14 00:31:53 INFO client.AsyncProcess: #20, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014
14/08/14 00:31:54 INFO client.AsyncProcess: #11, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014
14/08/14 00:31:57 INFO client.AsyncProcess: #7, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014
14/08/14 00:31:58 INFO client.AsyncProcess: #13, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014
14/08/14 00:31:58 INFO client.AsyncProcess: #12, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:29 PDT 2014
14/08/14 00:31:58 INFO client.AsyncProcess: #10, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:33 PDT 2014
2014-08-14 00:31:59:962 1891 sec: 11027175 operations; 5807.32 current ops/sec; [INSERT AverageLatency(us)=15174.74] [READ AverageLatency(us)=37246.84]
14/08/14 00:32:02 INFO client.AsyncProcess: #15, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014
14/08/14 00:32:02 INFO client.AsyncProcess: #5, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:34 PDT 2014
14/08/14 00:32:02 INFO client.AsyncProcess: #4, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:28:35 PDT 2014
2014-08-14 00:32:09:962 1901 sec: 11124329 operations; 9715.4 current ops/sec; [INSERT AverageLatency(us)=1761.82] [READ AverageLatency(us)=118339.98]
2014-08-14 00:32:19:976 1911 sec: 11231055 operations; 10657.68 current ops/sec; [INSERT AverageLatency(us)=1074.69] [READ AverageLatency(us)=52217.89]
2014-08-14 00:32:29:976 1921 sec: 11288064 operations; 5700.9 current ops/sec; [INSERT AverageLatency(us)=2530.35] [READ AverageLatency(us)=44721.97]
2014-08-14 00:32:39:977 1931 sec: 11376297 operations; 8822.42 current ops/sec; [INSERT AverageLatency(us)=902.58] [READ AverageLatency(us)=120870.16]
2014-08-14 00:32:49:977 1941 sec: 11376297 operations; 0 current ops/sec;
2014-08-14 00:32:59:978 1951 sec: 11376297 operations; 0 current ops/sec;
2014-08-14 00:33:09:978 1961 sec: 11376297 operations; 0 current ops/sec;
2014-08-14 00:33:19:979 1971 sec: 11376297 operations; 0 current ops/sec;
2014-08-14 00:33:29:980 1981 sec: 11376297 operations; 0 current ops/sec;
2014-08-14 00:33:39:981 1991 sec: 11376297 operations; 0 current ops/sec;
2014-08-14 00:33:49:982 2001 sec: 11376297 operations; 0 current ops/sec;
2014-08-14 00:33:59:982 2011 sec: 11377578 operations; 128.09 current ops/sec; [INSERT AverageLatency(us)=9.78] [READ AverageLatency(us)=16339004.3]
2014-08-14 00:34:09:983 2021 sec: 11377578 operations; 0 current ops/sec;
2014-08-14 00:34:19:985 2031 sec: 11472139 operations; 9455.15 current ops/sec; [INSERT AverageLatency(us)=19200.7] [READ AverageLatency(us)=80138.93]
2014-08-14 00:34:29:986 2041 sec: 11522810 operations; 5066.09 current ops/sec; [INSERT AverageLatency(us)=3941.98] [READ AverageLatency(us)=31942.18]
2014-08-14 00:34:39:987 2051 sec: 11638015 operations; 11519.35 current ops/sec; [INSERT AverageLatency(us)=1183.26] [READ AverageLatency(us)=41240.58]
2014-08-14 00:34:49:988 2061 sec: 11638015 operations; 0 current ops/sec;
2014-08-14 00:34:59:989 2071 sec: 11638015 operations; 0 current ops/sec;
2014-08-14 00:35:09:990 2081 sec: 11638015 operations; 0 current ops/sec;
2014-08-14 00:35:19:991 2091 sec: 11638015 operations; 0 current ops/sec;
2014-08-14 00:35:29:992 2101 sec: 11638015 operations; 0 current ops/sec;
2014-08-14 00:35:39:993 2111 sec: 11638015 operations; 0 current ops/sec;
2014-08-14 00:35:49:994 2121 sec: 11645182 operations; 716.63 current ops/sec; [INSERT AverageLatency(us)=10.39] [READ AverageLatency(us)=11441461.17]
2014-08-14 00:35:59:995 2131 sec: 11645182 operations; 0 current ops/sec;
2014-08-14 00:36:09:996 2141 sec: 11693701 operations; 4851.41 current ops/sec; [INSERT AverageLatency(us)=6026.72] [READ AverageLatency(us)=479878.84]
2014-08-14 00:36:19:998 2151 sec: 11772636 operations; 7892.71 current ops/sec; [INSERT AverageLatency(us)=6399.01] [READ AverageLatency(us)=86980.65]
2014-08-14 00:36:29:999 2161 sec: 11772636 operations; 0 current ops/sec;
2014-08-14 00:36:40:000 2171 sec: 11772636 operations; 0 current ops/sec;
2014-08-14 00:36:50:001 2181 sec: 11772636 operations; 0 current ops/sec;
2014-08-14 00:37:00:002 2191 sec: 11772636 operations; 0 current ops/sec;
2014-08-14 00:37:10:003 2201 sec: 11772636 operations; 0 current ops/sec;
2014-08-14 00:37:20:004 2211 sec: 11772636 operations; 0 current ops/sec;
2014-08-14 00:37:30:005 2221 sec: 11772636 operations; 0 current ops/sec;
2014-08-14 00:37:40:006 2231 sec: 11772636 operations; 0 current ops/sec;
2014-08-14 00:37:50:007 2241 sec: 11775114 operations; 247.78 current ops/sec; [INSERT AverageLatency(us)=11.3] [READ AverageLatency(us)=89523329.6]
2014-08-14 00:38:00:008 2251 sec: 11830449 operations; 5532.95 current ops/sec; [INSERT AverageLatency(us)=12.69] [READ AverageLatency(us)=466010.89]
2014-08-14 00:38:10:009 2261 sec: 11839585 operations; 913.51 current ops/sec; [INSERT AverageLatency(us)=18697.95] [READ AverageLatency(us)=115057.34]
2014-08-14 00:38:20:011 2271 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:38:30:012 2281 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:38:40:013 2291 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:38:50:014 2301 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:39:00:014 2311 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:39:10:015 2321 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:39:20:016 2331 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:39:30:017 2341 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:39:40:018 2351 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:39:50:019 2361 sec: 11839585 operations; 0 current ops/sec;
2014-08-14 00:40:00:019 2371 sec: 11931791 operations; 9220.6 current ops/sec; [INSERT AverageLatency(us)=10.75] [READ AverageLatency(us)=2492152.84]
2014-08-14 00:40:10:020 2381 sec: 11948918 operations; 1712.53 current ops/sec; [INSERT AverageLatency(us)=674.36] [READ AverageLatency(us)=27297.61]
2014-08-14 00:40:20:022 2391 sec: 11948918 operations; 0 current ops/sec;
2014-08-14 00:40:30:023 2401 sec: 11992097 operations; 4317.47 current ops/sec; [INSERT AverageLatency(us)=2527.53] [READ AverageLatency(us)=519639.51]
2014-08-14 00:40:40:024 2411 sec: 12020380 operations; 2828.02 current ops/sec; [INSERT AverageLatency(us)=4993.71] [READ AverageLatency(us)=39703.12]
2014-08-14 00:40:50:025 2421 sec: 12020380 operations; 0 current ops/sec;
2014-08-14 00:41:00:026 2431 sec: 12065125 operations; 4474.05 current ops/sec; [INSERT AverageLatency(us)=9910.87] [READ AverageLatency(us)=449569.59]
2014-08-14 00:41:10:027 2441 sec: 12065125 operations; 0 current ops/sec;
2014-08-14 00:41:20:028 2451 sec: 12211020 operations; 14588.04 current ops/sec; [INSERT AverageLatency(us)=609.62] [READ AverageLatency(us)=239839.73]
2014-08-14 00:41:30:029 2461 sec: 12211020 operations; 0 current ops/sec;
2014-08-14 00:41:40:030 2471 sec: 12348100 operations; 13706.63 current ops/sec; [INSERT AverageLatency(us)=1404.26] [READ AverageLatency(us)=182576.78]
2014-08-14 00:41:50:031 2481 sec: 12453725 operations; 10561.44 current ops/sec; [INSERT AverageLatency(us)=998.54] [READ AverageLatency(us)=42424.49]
2014-08-14 00:42:00:032 2491 sec: 12453725 operations; 0 current ops/sec;
2014-08-14 00:42:10:033 2501 sec: 12547331 operations; 9359.66 current ops/sec; [INSERT AverageLatency(us)=2427.28] [READ AverageLatency(us)=247854.66]
2014-08-14 00:42:20:035 2511 sec: 12627701 operations; 8035.39 current ops/sec; [INSERT AverageLatency(us)=68.51] [READ AverageLatency(us)=52220.21]
2014-08-14 00:42:30:036 2521 sec: 12646840 operations; 1913.71 current ops/sec; [INSERT AverageLatency(us)=3483.95] [READ AverageLatency(us)=140667.36]
2014-08-14 00:42:40:038 2531 sec: 12779026 operations; 13215.96 current ops/sec; [INSERT AverageLatency(us)=2230.87] [READ AverageLatency(us)=66715.63]
2014-08-14 00:42:50:039 2541 sec: 12892913 operations; 11387.56 current ops/sec; [INSERT AverageLatency(us)=1256.34] [READ AverageLatency(us)=86317.63]
2014-08-14 00:43:00:040 2551 sec: 12915764 operations; 2284.87 current ops/sec; [INSERT AverageLatency(us)=8525.37] [READ AverageLatency(us)=84678.33]
2014-08-14 00:43:10:041 2561 sec: 13033812 operations; 11803.62 current ops/sec; [INSERT AverageLatency(us)=756.9] [READ AverageLatency(us)=43186.88]
2014-08-14 00:43:20:042 2571 sec: 13116146 operations; 8232.58 current ops/sec; [INSERT AverageLatency(us)=2150.23] [READ AverageLatency(us)=56659.74]
2014-08-14 00:43:30:043 2581 sec: 13172710 operations; 5655.83 current ops/sec; [INSERT AverageLatency(us)=1341.6] [READ AverageLatency(us)=29602.51]
2014-08-14 00:43:40:044 2591 sec: 13286459 operations; 11373.76 current ops/sec; [INSERT AverageLatency(us)=1897.3] [READ AverageLatency(us)=61411.02]
2014-08-14 00:43:50:045 2601 sec: 13371290 operations; 8482.25 current ops/sec; [INSERT AverageLatency(us)=2799.25] [READ AverageLatency(us)=68585.32]
2014-08-14 00:44:00:235 2611 sec: 13421389 operations; 4916.49 current ops/sec; [INSERT AverageLatency(us)=1585.27] [READ AverageLatency(us)=63891.52]
2014-08-14 00:44:10:236 2621 sec: 13518969 operations; 9757.02 current ops/sec; [INSERT AverageLatency(us)=1334.28] [READ AverageLatency(us)=55078.44]
2014-08-14 00:44:20:237 2631 sec: 13618625 operations; 9964.6 current ops/sec; [INSERT AverageLatency(us)=1881.24] [READ AverageLatency(us)=50166.49]
2014-08-14 00:44:30:238 2641 sec: 13618625 operations; 0 current ops/sec;
2014-08-14 00:44:40:239 2651 sec: 13618625 operations; 0 current ops/sec;
2014-08-14 00:44:50:240 2661 sec: 13618625 operations; 0 current ops/sec;
2014-08-14 00:45:00:241 2671 sec: 13651599 operations; 3297.07 current ops/sec; [INSERT AverageLatency(us)=9376.38] [READ AverageLatency(us)=24655.72]
2014-08-14 00:45:10:243 2681 sec: 13751431 operations; 9982.2 current ops/sec; [INSERT AverageLatency(us)=6657.03] [READ AverageLatency(us)=43971.8]
2014-08-14 00:45:20:244 2691 sec: 13792426 operations; 4098.68 current ops/sec; [INSERT AverageLatency(us)=4823.9] [READ AverageLatency(us)=91780.13]
2014-08-14 00:45:30:245 2701 sec: 13907943 operations; 11550.54 current ops/sec; [INSERT AverageLatency(us)=802.37] [READ AverageLatency(us)=19598.23]
2014-08-14 00:45:40:246 2711 sec: 13985505 operations; 7755.42 current ops/sec; [INSERT AverageLatency(us)=2882.28] [READ AverageLatency(us)=51519.71]
2014-08-14 00:45:50:247 2721 sec: 14056627 operations; 7111.49 current ops/sec; [INSERT AverageLatency(us)=2705.65] [READ AverageLatency(us)=35821.11]
2014-08-14 00:46:00:248 2731 sec: 14154512 operations; 9787.52 current ops/sec; [INSERT AverageLatency(us)=856.81] [READ AverageLatency(us)=48886.65]
2014-08-14 00:46:10:249 2741 sec: 14220964 operations; 6644.54 current ops/sec; [INSERT AverageLatency(us)=2054.84] [READ AverageLatency(us)=34909.75]
2014-08-14 00:46:20:251 2751 sec: 14320520 operations; 9954.6 current ops/sec; [INSERT AverageLatency(us)=2149.11] [READ AverageLatency(us)=62039.87]
2014-08-14 00:46:30:252 2761 sec: 14380639 operations; 6010.7 current ops/sec; [INSERT AverageLatency(us)=3130.58] [READ AverageLatency(us)=72441.4]
2014-08-14 00:46:40:256 2771 sec: 14380639 operations; 0 current ops/sec;
2014-08-14 00:46:50:257 2781 sec: 14380639 operations; 0 current ops/sec;
2014-08-14 00:47:00:257 2791 sec: 14380639 operations; 0 current ops/sec;
2014-08-14 00:47:10:258 2801 sec: 14401690 operations; 2104.89 current ops/sec; [INSERT AverageLatency(us)=8.87] [READ AverageLatency(us)=2630786.65]
2014-08-14 00:47:20:260 2811 sec: 14464666 operations; 6296.34 current ops/sec; [INSERT AverageLatency(us)=3764.99] [READ AverageLatency(us)=66019]
2014-08-14 00:47:30:261 2821 sec: 14558171 operations; 9349.57 current ops/sec; [INSERT AverageLatency(us)=2073.59] [READ AverageLatency(us)=69555.59]
2014-08-14 00:47:40:262 2831 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:47:50:263 2841 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:48:00:264 2851 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:48:10:265 2861 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:48:20:266 2871 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:48:30:267 2881 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:48:40:268 2891 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:48:50:269 2901 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:49:00:270 2911 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:49:10:271 2921 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:49:20:272 2931 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:49:30:272 2941 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:49:40:273 2951 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:49:50:274 2961 sec: 14558171 operations; 0 current ops/sec;
14/08/14 00:49:54 INFO client.AsyncProcess: #6, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10054 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #19, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:23 PDT 2014, retrying after 10046 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #18, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10030 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #14, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10007 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #15, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10018 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #1, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10042 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #11, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10035 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #3, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:20 PDT 2014, retrying after 10061 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #9, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10055 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #10, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:20 PDT 2014, retrying after 10054 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #17, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10056 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #5, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:23 PDT 2014, retrying after 10032 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #8, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10004 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #13, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10064 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #2, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10027 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #7, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10032 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #16, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10003 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #20, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10099 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #4, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10066 ms, replay 4696 ops.
14/08/14 00:49:54 INFO client.AsyncProcess: #12, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10084 ms, replay 4696 ops.
2014-08-14 00:50:00:275 2971 sec: 14558171 operations; 0 current ops/sec;
14/08/14 00:50:04 INFO client.AsyncProcess: #6, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10068 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #19, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:23 PDT 2014, retrying after 10074 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #14, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10050 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #18, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10063 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #15, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10090 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #1, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10008 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #11, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10075 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #8, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10052 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #3, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:20 PDT 2014, retrying after 10045 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #5, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:23 PDT 2014, retrying after 10096 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #2, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10014 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #16, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10031 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #9, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10099 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #10, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:20 PDT 2014, retrying after 10062 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #17, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10098 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #7, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10016 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #13, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10071 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #20, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10014 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #4, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 10046 ms, replay 4696 ops.
14/08/14 00:50:04 INFO client.AsyncProcess: #12, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 10012 ms, replay 4696 ops.
2014-08-14 00:50:10:276 2981 sec: 14558171 operations; 0 current ops/sec;
14/08/14 00:50:14 INFO client.AsyncProcess: #19, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:23 PDT 2014, retrying after 20151 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #6, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20027 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #14, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 20020 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #18, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 20182 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #1, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20172 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #15, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 20138 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #8, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20193 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #11, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20097 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #2, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20088 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #7, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 20031 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #16, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20102 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #3, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:20 PDT 2014, retrying after 20014 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #10, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:20 PDT 2014, retrying after 20005 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #5, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:23 PDT 2014, retrying after 20066 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #9, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20130 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #13, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 20163 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #17, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 20198 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #20, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20195 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #12, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014, retrying after 20188 ms, replay 4696 ops.
14/08/14 00:50:14 INFO client.AsyncProcess: #4, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36592 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014, retrying after 20171 ms, replay 4696 ops.
2014-08-14 00:50:20:277 2991 sec: 14558171 operations; 0 current ops/sec;
2014-08-14 00:50:30:278 3001 sec: 14558171 operations; 0 current ops/sec;
14/08/14 00:50:35 INFO client.AsyncProcess: #6, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
14/08/14 00:50:35 INFO client.AsyncProcess: #14, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014
14/08/14 00:50:35 INFO client.AsyncProcess: #10, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:20 PDT 2014
14/08/14 00:50:35 INFO client.AsyncProcess: #3, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:20 PDT 2014
14/08/14 00:50:35 INFO client.AsyncProcess: #7, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014
14/08/14 00:50:36 INFO client.AsyncProcess: #19, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:23 PDT 2014
14/08/14 00:50:36 INFO client.AsyncProcess: #4, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
14/08/14 00:50:36 INFO client.AsyncProcess: #20, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
14/08/14 00:50:36 INFO client.AsyncProcess: #12, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014
14/08/14 00:50:37 INFO client.AsyncProcess: #17, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014
14/08/14 00:50:37 INFO client.AsyncProcess: #13, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014
14/08/14 00:50:37 INFO client.AsyncProcess: #9, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
14/08/14 00:50:37 INFO client.AsyncProcess: #8, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
14/08/14 00:50:37 INFO client.AsyncProcess: #5, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:23 PDT 2014
14/08/14 00:50:37 INFO client.AsyncProcess: #15, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014
14/08/14 00:50:38 INFO client.AsyncProcess: #16, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
2014-08-14 00:50:40:278 3011 sec: 14625242 operations; 6707.1 current ops/sec; [INSERT AverageLatency(us)=23321.97] [READ AverageLatency(us)=2699958.39]
14/08/14 00:50:43 INFO client.AsyncProcess: #18, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:24 PDT 2014
14/08/14 00:50:43 INFO client.AsyncProcess: #11, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
14/08/14 00:50:43 INFO client.AsyncProcess: #2, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
14/08/14 00:50:43 INFO client.AsyncProcess: #1, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:47:21 PDT 2014
2014-08-14 00:50:50:279 3021 sec: 14699970 operations; 7472.05 current ops/sec; [INSERT AverageLatency(us)=8798.45] [READ AverageLatency(us)=28185.92]
2014-08-14 00:51:00:280 3031 sec: 14799594 operations; 9961.4 current ops/sec; [INSERT AverageLatency(us)=2009.65] [READ AverageLatency(us)=54601.17]
2014-08-14 00:51:10:281 3041 sec: 14853646 operations; 5404.66 current ops/sec; [INSERT AverageLatency(us)=2996.33] [READ AverageLatency(us)=33902.65]
2014-08-14 00:51:20:282 3051 sec: 14853646 operations; 0 current ops/sec;
2014-08-14 00:51:30:283 3061 sec: 14853646 operations; 0 current ops/sec;
2014-08-14 00:51:40:288 3071 sec: 14853646 operations; 0 current ops/sec;
2014-08-14 00:51:50:289 3081 sec: 14853646 operations; 0 current ops/sec;
2014-08-14 00:52:00:290 3091 sec: 14853646 operations; 0 current ops/sec;
2014-08-14 00:52:10:291 3101 sec: 14853646 operations; 0 current ops/sec;
2014-08-14 00:52:20:292 3111 sec: 14853646 operations; 0 current ops/sec;
2014-08-14 00:52:30:293 3121 sec: 14854483 operations; 83.69 current ops/sec; [INSERT AverageLatency(us)=11.25] [READ AverageLatency(us)=81408261.67]
2014-08-14 00:52:40:294 3131 sec: 14899438 operations; 4495.05 current ops/sec; [INSERT AverageLatency(us)=4272.76] [READ AverageLatency(us)=180677.33]
2014-08-14 00:52:50:296 3141 sec: 14971628 operations; 7217.56 current ops/sec; [INSERT AverageLatency(us)=12868.46] [READ AverageLatency(us)=60074.67]
2014-08-14 00:53:00:297 3151 sec: 14971628 operations; 0 current ops/sec;
2014-08-14 00:53:10:298 3161 sec: 15022774 operations; 5114.09 current ops/sec; [INSERT AverageLatency(us)=3950.26] [READ AverageLatency(us)=175450.97]
2014-08-14 00:53:20:299 3171 sec: 15131644 operations; 10885.91 current ops/sec; [INSERT AverageLatency(us)=2842.37] [READ AverageLatency(us)=50117.95]
2014-08-14 00:53:30:300 3181 sec: 15217029 operations; 8537.65 current ops/sec; [INSERT AverageLatency(us)=1838.29] [READ AverageLatency(us)=51504.67]
2014-08-14 00:53:40:301 3191 sec: 15297652 operations; 8061.49 current ops/sec; [INSERT AverageLatency(us)=1763.84] [READ AverageLatency(us)=40349.83]
2014-08-14 00:53:50:302 3201 sec: 15373138 operations; 7547.85 current ops/sec; [INSERT AverageLatency(us)=2625.62] [READ AverageLatency(us)=59492.48]
2014-08-14 00:54:00:303 3211 sec: 15411481 operations; 3833.92 current ops/sec; [INSERT AverageLatency(us)=5942.85] [READ AverageLatency(us)=34685.45]
2014-08-14 00:54:10:304 3221 sec: 15520590 operations; 10909.81 current ops/sec; [INSERT AverageLatency(us)=122.58] [READ AverageLatency(us)=68820.4]
2014-08-14 00:54:20:305 3231 sec: 15620224 operations; 9962.4 current ops/sec; [INSERT AverageLatency(us)=2236.17] [READ AverageLatency(us)=52419.19]
2014-08-14 00:54:30:306 3241 sec: 15716225 operations; 9599.14 current ops/sec; [INSERT AverageLatency(us)=1835.26] [READ AverageLatency(us)=37924.75]
2014-08-14 00:54:40:308 3251 sec: 15754573 operations; 3834.03 current ops/sec; [INSERT AverageLatency(us)=2695.66] [READ AverageLatency(us)=36564.05]
2014-08-14 00:54:50:309 3261 sec: 15847895 operations; 9331.27 current ops/sec; [INSERT AverageLatency(us)=1837.3] [READ AverageLatency(us)=34833.98]
2014-08-14 00:55:00:310 3271 sec: 15931355 operations; 8345.17 current ops/sec; [INSERT AverageLatency(us)=3030.54] [READ AverageLatency(us)=82813]
2014-08-14 00:55:10:311 3281 sec: 16009181 operations; 7781.82 current ops/sec; [INSERT AverageLatency(us)=839.63] [READ AverageLatency(us)=24555.55]
2014-08-14 00:55:20:312 3291 sec: 16089155 operations; 7996.6 current ops/sec; [INSERT AverageLatency(us)=2823.47] [READ AverageLatency(us)=71510.15]
2014-08-14 00:55:30:313 3301 sec: 16135002 operations; 4584.24 current ops/sec; [INSERT AverageLatency(us)=3564.2] [READ AverageLatency(us)=26530.11]
2014-08-14 00:55:40:315 3311 sec: 16138307 operations; 330.43 current ops/sec; [INSERT AverageLatency(us)=18766.07] [READ AverageLatency(us)=764640.3]
2014-08-14 00:55:50:316 3321 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:56:00:317 3331 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:56:10:318 3341 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:56:20:319 3351 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:56:30:320 3361 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:56:40:321 3371 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:56:50:322 3381 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:57:00:331 3391 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:57:10:332 3401 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:57:20:333 3411 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:57:30:334 3421 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:57:40:334 3431 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:57:50:335 3441 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:58:00:336 3451 sec: 16138307 operations; 0 current ops/sec;
2014-08-14 00:58:10:337 3461 sec: 16139949 operations; 164.18 current ops/sec; [INSERT AverageLatency(us)=11.25] [READ AverageLatency(us)=158358009.89]
2014-08-14 00:58:20:339 3471 sec: 16145566 operations; 561.64 current ops/sec; [INSERT AverageLatency(us)=8.2] [READ AverageLatency(us)=1006851.46]
2014-08-14 00:58:30:340 3481 sec: 16217198 operations; 7161.77 current ops/sec; [INSERT AverageLatency(us)=11.19] [READ AverageLatency(us)=458837.28]
2014-08-14 00:58:40:341 3491 sec: 16248254 operations; 3105.29 current ops/sec; [INSERT AverageLatency(us)=6673.67] [READ AverageLatency(us)=31063.2]
2014-08-14 00:58:50:343 3501 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 00:59:00:344 3511 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 00:59:10:345 3521 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 00:59:20:346 3531 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 00:59:30:346 3541 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 00:59:40:347 3551 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 00:59:50:348 3561 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 01:00:00:349 3571 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 01:00:10:350 3581 sec: 16248254 operations; 0 current ops/sec;
2014-08-14 01:00:20:351 3591 sec: 16272021 operations; 2376.46 current ops/sec; [INSERT AverageLatency(us)=10.15] [READ AverageLatency(us)=6438307.18]
2014-08-14 01:00:30:352 3601 sec: 16332048 operations; 6002.1 current ops/sec; [INSERT AverageLatency(us)=1888.72] [READ AverageLatency(us)=47262.76]
2014-08-14 01:00:40:353 3611 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:00:50:355 3621 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:01:00:355 3631 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:01:10:357 3641 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:01:20:358 3651 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:01:30:359 3661 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:01:40:360 3671 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:01:50:361 3681 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:02:00:362 3691 sec: 16332048 operations; 0 current ops/sec;
2014-08-14 01:02:10:363 3701 sec: 16332364 operations; 31.6 current ops/sec; [INSERT AverageLatency(us)=8.21] [READ AverageLatency(us)=102318477]
2014-08-14 01:02:20:364 3711 sec: 16336675 operations; 431.06 current ops/sec; [INSERT AverageLatency(us)=27061.79] [READ AverageLatency(us)=590524.93]
2014-08-14 01:02:30:365 3721 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:02:40:367 3731 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:02:50:372 3741 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:03:00:373 3751 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:03:10:374 3761 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:03:20:375 3771 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:03:30:376 3781 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:03:40:380 3791 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:03:50:381 3801 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:04:00:382 3811 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:04:10:383 3821 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:04:20:383 3831 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:04:30:384 3841 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:04:40:385 3851 sec: 16336675 operations; 0 current ops/sec;
14/08/14 01:04:46 INFO client.AsyncProcess: #13, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10071 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #10, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10022 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #9, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10009 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #2, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10091 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #11, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10052 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #7, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10089 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #18, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10058 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #1, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 10016 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #14, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 10068 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #19, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10087 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #6, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 10037 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #4, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 10039 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #12, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10011 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #5, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10001 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #8, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10070 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #3, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10003 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #15, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:23 PDT 2014, retrying after 10011 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #17, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:21 PDT 2014, retrying after 10032 ms, replay 4696 ops.
14/08/14 01:04:46 INFO client.AsyncProcess: #20, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:22 PDT 2014, retrying after 10098 ms, replay 4696 ops.
14/08/14 01:04:47 INFO client.AsyncProcess: #16, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:02:18 PDT 2014, retrying after 10023 ms, replay 4696 ops.
2014-08-14 01:04:50:386 3861 sec: 16336675 operations; 0 current ops/sec;
14/08/14 01:04:56 INFO client.AsyncProcess: #9, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10067 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #10, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10028 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #13, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10080 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #2, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10038 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #1, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 10055 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #11, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10035 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #12, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10071 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #18, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10027 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #5, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10049 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #6, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 10069 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #3, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10030 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #4, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 10016 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #8, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10082 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #19, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10006 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #7, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 10052 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #14, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 10075 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #15, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:23 PDT 2014, retrying after 10079 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #17, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:21 PDT 2014, retrying after 10084 ms, replay 4696 ops.
14/08/14 01:04:56 INFO client.AsyncProcess: #20, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:22 PDT 2014, retrying after 10078 ms, replay 4696 ops.
14/08/14 01:04:57 INFO client.AsyncProcess: #16, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:02:18 PDT 2014, retrying after 10024 ms, replay 4696 ops.
2014-08-14 01:05:00:387 3871 sec: 16336675 operations; 0 current ops/sec;
14/08/14 01:05:06 INFO client.AsyncProcess: #10, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20109 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #9, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20044 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #2, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20115 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #11, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20119 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #13, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20033 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #1, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 20120 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #4, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 20151 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #19, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20028 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #18, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20122 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #3, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20010 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #5, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20102 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #6, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 20023 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #7, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20076 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #12, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20022 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #14, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014, retrying after 20046 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #8, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014, retrying after 20056 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #15, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:23 PDT 2014, retrying after 20033 ms, replay 4696 ops.
14/08/14 01:05:06 INFO client.AsyncProcess: #17, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:21 PDT 2014, retrying after 20006 ms, replay 4696 ops.
14/08/14 01:05:07 INFO client.AsyncProcess: #20, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:22 PDT 2014, retrying after 20184 ms, replay 4696 ops.
14/08/14 01:05:07 INFO client.AsyncProcess: #16, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36607 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:02:18 PDT 2014, retrying after 20068 ms, replay 4696 ops.
2014-08-14 01:05:10:388 3881 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:05:20:389 3891 sec: 16336675 operations; 0 current ops/sec;
2014-08-14 01:05:30:390 3901 sec: 16336741 operations; 6.6 current ops/sec; [INSERT AverageLatency(us)=10.73] [READ AverageLatency(us)=63057828.33]
14/08/14 01:05:31 INFO client.AsyncProcess: #8, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #13, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #4, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #10, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #9, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #5, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #14, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #7, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #18, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #1, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #12, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #6, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:30 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #11, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:31 INFO client.AsyncProcess: #2, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:35 INFO client.AsyncProcess: #3, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:35 INFO client.AsyncProcess: #19, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 00:58:31 PDT 2014
14/08/14 01:05:35 INFO client.AsyncProcess: #16, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:02:18 PDT 2014
14/08/14 01:05:35 INFO client.AsyncProcess: #20, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:22 PDT 2014
14/08/14 01:05:35 INFO client.AsyncProcess: #15, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:23 PDT 2014
14/08/14 01:05:35 INFO client.AsyncProcess: #17, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:00:21 PDT 2014
2014-08-14 01:05:40:391 3911 sec: 16419442 operations; 8269.27 current ops/sec; [INSERT AverageLatency(us)=72205.01] [READ AverageLatency(us)=421514.5]
2014-08-14 01:05:50:392 3921 sec: 16419442 operations; 0 current ops/sec;
2014-08-14 01:06:00:393 3931 sec: 16419442 operations; 0 current ops/sec;
2014-08-14 01:06:10:394 3941 sec: 16419442 operations; 0 current ops/sec;
2014-08-14 01:06:20:395 3951 sec: 16579902 operations; 16044.4 current ops/sec; [INSERT AverageLatency(us)=96.98] [READ AverageLatency(us)=463698.1]
2014-08-14 01:06:30:397 3961 sec: 16583076 operations; 317.34 current ops/sec; [INSERT AverageLatency(us)=1123.43] [READ AverageLatency(us)=14507.94]
2014-08-14 01:06:40:398 3971 sec: 16601792 operations; 1871.41 current ops/sec; [INSERT AverageLatency(us)=1148.54] [READ AverageLatency(us)=460673.45]
2014-08-14 01:06:50:399 3981 sec: 16611683 operations; 989 current ops/sec; [INSERT AverageLatency(us)=39869.72] [READ AverageLatency(us)=617518.94]
2014-08-14 01:07:00:400 3991 sec: 16611683 operations; 0 current ops/sec;
2014-08-14 01:07:10:401 4001 sec: 16782532 operations; 17083.19 current ops/sec; [INSERT AverageLatency(us)=18.34] [READ AverageLatency(us)=275289.07]
2014-08-14 01:07:20:402 4011 sec: 16887454 operations; 10491.15 current ops/sec; [INSERT AverageLatency(us)=633.48] [READ AverageLatency(us)=69712.88]
2014-08-14 01:07:30:403 4021 sec: 16887454 operations; 0 current ops/sec;
2014-08-14 01:07:40:404 4031 sec: 16977586 operations; 9012.3 current ops/sec; [INSERT AverageLatency(us)=2320.77] [READ AverageLatency(us)=295906.88]
2014-08-14 01:07:50:405 4041 sec: 17118343 operations; 14074.29 current ops/sec; [INSERT AverageLatency(us)=622.34] [READ AverageLatency(us)=89154.45]
2014-08-14 01:08:00:430 4051 sec: 17290818 operations; 17204.49 current ops/sec; [INSERT AverageLatency(us)=459.74] [READ AverageLatency(us)=55504.19]
2014-08-14 01:08:10:436 4061 sec: 17291696 operations; 87.75 current ops/sec; [INSERT AverageLatency(us)=24949.62] [READ AverageLatency(us)=269281]
2014-08-14 01:08:20:438 4071 sec: 17291696 operations; 0 current ops/sec;
2014-08-14 01:08:30:439 4081 sec: 17291696 operations; 0 current ops/sec;
2014-08-14 01:08:40:440 4091 sec: 17291696 operations; 0 current ops/sec;
2014-08-14 01:08:50:441 4101 sec: 17414570 operations; 12286.17 current ops/sec; [INSERT AverageLatency(us)=206.28] [READ AverageLatency(us)=730277.28]
2014-08-14 01:09:00:442 4111 sec: 17557952 operations; 14336.77 current ops/sec; [INSERT AverageLatency(us)=767.33] [READ AverageLatency(us)=66077.46]
2014-08-14 01:09:10:443 4121 sec: 17576757 operations; 1880.31 current ops/sec; [INSERT AverageLatency(us)=3080.43] [READ AverageLatency(us)=176909.2]
2014-08-14 01:09:20:445 4131 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:09:30:446 4141 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:09:40:447 4151 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:09:50:448 4161 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:10:00:449 4171 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:10:10:450 4181 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:10:20:451 4191 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:10:30:452 4201 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:10:40:453 4211 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:10:50:454 4221 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:11:00:455 4231 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:11:10:455 4241 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:11:20:456 4251 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:11:30:458 4261 sec: 17576757 operations; 0 current ops/sec;
14/08/14 01:11:31 INFO client.AsyncProcess: #8, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10016 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #20, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10007 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #2, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10083 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #1, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 10005 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #6, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10034 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #13, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10096 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #17, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:53 PDT 2014, retrying after 10034 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #14, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 10071 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #3, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:55 PDT 2014, retrying after 10012 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #10, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10023 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #19, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:54 PDT 2014, retrying after 10077 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #7, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 10023 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #16, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10015 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #4, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10081 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #9, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:59 PDT 2014, retrying after 10023 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #5, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10007 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #11, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10078 ms, replay 4696 ops.
14/08/14 01:11:31 INFO client.AsyncProcess: #15, table=usertable, attempt=10/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10027 ms, replay 4696 ops.
2014-08-14 01:11:40:459 4271 sec: 17576757 operations; 0 current ops/sec;
14/08/14 01:11:41 INFO client.AsyncProcess: #8, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10058 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #20, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10084 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #1, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 10012 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #6, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10062 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #3, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:55 PDT 2014, retrying after 10085 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #2, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10079 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #17, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:53 PDT 2014, retrying after 10047 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #14, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 10014 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #7, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 10096 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #10, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10056 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #5, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10017 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #16, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10057 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #9, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:59 PDT 2014, retrying after 10037 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #13, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10070 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #15, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10073 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #19, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:54 PDT 2014, retrying after 10097 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #4, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10025 ms, replay 4696 ops.
14/08/14 01:11:41 INFO client.AsyncProcess: #11, table=usertable, attempt=11/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 10039 ms, replay 4696 ops.
2014-08-14 01:11:50:460 4281 sec: 17576757 operations; 0 current ops/sec;
14/08/14 01:11:51 INFO client.AsyncProcess: #8, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20197 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #1, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 20163 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #20, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20192 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #5, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20128 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #14, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 20017 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #17, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:53 PDT 2014, retrying after 20036 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #6, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20075 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #3, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:55 PDT 2014, retrying after 20157 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #2, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20117 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #9, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:59 PDT 2014, retrying after 20168 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #10, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20071 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #16, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20048 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #13, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20092 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #4, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20017 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #7, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014, retrying after 20181 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #15, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20046 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #11, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014, retrying after 20061 ms, replay 4696 ops.
14/08/14 01:11:51 INFO client.AsyncProcess: #19, table=usertable, attempt=12/35 failed 4696 ops, last exception: java.net.SocketTimeoutException: Call to slave1/9.1.143.59:16020 failed because java.net.SocketTimeoutException: 60000 millis timeout while waiting for channel to be ready for write. ch : java.nio.channels.SocketChannel[connected local=/9.1.143.53:36612 remote=slave1/9.1.143.59:16020] on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:54 PDT 2014, retrying after 20060 ms, replay 4696 ops.
2014-08-14 01:12:00:460 4291 sec: 17576757 operations; 0 current ops/sec;
2014-08-14 01:12:10:461 4301 sec: 17576757 operations; 0 current ops/sec;
14/08/14 01:12:12 INFO client.AsyncProcess: #14, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014
14/08/14 01:12:12 INFO client.AsyncProcess: #10, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:15 INFO client.AsyncProcess: #4, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:15 INFO client.AsyncProcess: #16, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:15 INFO client.AsyncProcess: #6, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:15 INFO client.AsyncProcess: #8, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:15 INFO client.AsyncProcess: #1, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014
14/08/14 01:12:16 INFO client.AsyncProcess: #17, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:53 PDT 2014
14/08/14 01:12:16 INFO client.AsyncProcess: #7, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:00 PDT 2014
14/08/14 01:12:16 INFO client.AsyncProcess: #19, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:54 PDT 2014
14/08/14 01:12:16 INFO client.AsyncProcess: #9, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:59 PDT 2014
14/08/14 01:12:17 INFO client.AsyncProcess: #3, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:08:55 PDT 2014
14/08/14 01:12:17 INFO client.AsyncProcess: #11, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:17 INFO client.AsyncProcess: #15, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:17 INFO client.AsyncProcess: #13, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:17 INFO client.AsyncProcess: #2, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:18 INFO client.AsyncProcess: #20, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
14/08/14 01:12:18 INFO client.AsyncProcess: #5, table=usertable, attempt=13/35 SUCCEEDED on slave1,16020,1408015324622, tracking started Thu Aug 14 01:09:01 PDT 2014
2014-08-14 01:12:20:462 4311 sec: 17655510 operations; 7874.51 current ops/sec; [INSERT AverageLatency(us)=7886.9] [READ AverageLatency(us)=4252482.32]
2014-08-14 01:12:30:463 4321 sec: 17768652 operations; 11313.07 current ops/sec; [INSERT AverageLatency(us)=101.54] [READ AverageLatency(us)=57608.19]
2014-08-14 01:12:40:464 4331 sec: 17768652 operations; 0 current ops/sec;
2014-08-14 01:12:50:465 4341 sec: 17768652 operations; 0 current ops/sec;
2014-08-14 01:13:00:466 4351 sec: 17768652 operations; 0 current ops/sec;
2014-08-14 01:13:10:467 4361 sec: 17892285 operations; 12362.06 current ops/sec; [INSERT AverageLatency(us)=6040.5] [READ AverageLatency(us)=135776.29]
2014-08-14 01:13:20:468 4371 sec: 18033156 operations; 14085.69 current ops/sec; [INSERT AverageLatency(us)=818.54] [READ AverageLatency(us)=67618.4]
2014-08-14 01:13:30:470 4381 sec: 18044483 operations; 1132.47 current ops/sec; [INSERT AverageLatency(us)=267.34] [READ AverageLatency(us)=31879.92]
2014-08-14 01:13:40:471 4391 sec: 18102819 operations; 5833.02 current ops/sec; [INSERT AverageLatency(us)=2724.82] [READ AverageLatency(us)=255145.82]
2014-08-14 01:13:50:472 4401 sec: 18209491 operations; 10666.13 current ops/sec; [INSERT AverageLatency(us)=1431.93] [READ AverageLatency(us)=104014.1]
2014-08-14 01:14:00:473 4411 sec: 18325339 operations; 11583.64 current ops/sec; [INSERT AverageLatency(us)=1217.06] [READ AverageLatency(us)=57936.42]
2014-08-14 01:14:10:474 4421 sec: 18477243 operations; 15188.88 current ops/sec; [INSERT AverageLatency(us)=716.06] [READ AverageLatency(us)=55989.66]
2014-08-14 01:14:20:476 4431 sec: 18563083 operations; 8582.28 current ops/sec; [INSERT AverageLatency(us)=200.52] [READ AverageLatency(us)=84528.43]
2014-08-14 01:14:30:477 4441 sec: 18563083 operations; 0 current ops/sec;
2014-08-14 01:14:40:478 4451 sec: 18670848 operations; 10775.42 current ops/sec; [INSERT AverageLatency(us)=1636.83] [READ AverageLatency(us)=316980.63]
2014-08-14 01:14:50:479 4461 sec: 18818868 operations; 14800.52 current ops/sec; [INSERT AverageLatency(us)=694.07] [READ AverageLatency(us)=56539.75]
2014-08-14 01:15:00:479 4471 sec: 18973690 operations; 15482.2 current ops/sec; [INSERT AverageLatency(us)=515.97] [READ AverageLatency(us)=68372.53]
2014-08-14 01:15:10:480 4481 sec: 19031289 operations; 5759.32 current ops/sec; [INSERT AverageLatency(us)=1289.47] [READ AverageLatency(us)=43092.27]
2014-08-14 01:15:20:481 4491 sec: 19031289 operations; 0 current ops/sec;
2014-08-14 01:15:30:482 4501 sec: 19112151 operations; 8085.39 current ops/sec; [INSERT AverageLatency(us)=5034.6] [READ AverageLatency(us)=146955.09]
2014-08-14 01:15:40:483 4511 sec: 19180116 operations; 6795.82 current ops/sec; [INSERT AverageLatency(us)=1833.93] [READ AverageLatency(us)=102557.25]
2014-08-14 01:15:50:484 4521 sec: 19180116 operations; 0 current ops/sec;
2014-08-14 01:16:00:485 4531 sec: 19266377 operations; 8625.24 current ops/sec; [INSERT AverageLatency(us)=355.18] [READ AverageLatency(us)=387335.97]
2014-08-14 01:16:10:486 4541 sec: 19401273 operations; 13488.25 current ops/sec; [INSERT AverageLatency(us)=1038.13] [READ AverageLatency(us)=73315.57]
2014-08-14 01:16:20:487 4551 sec: 19517438 operations; 11615.34 current ops/sec; [INSERT AverageLatency(us)=494.2] [READ AverageLatency(us)=58207.87]
2014-08-14 01:16:30:489 4561 sec: 19517438 operations; 0 current ops/sec;
2014-08-14 01:16:40:490 4571 sec: 19517438 operations; 0 current ops/sec;
2014-08-14 01:16:50:491 4581 sec: 19550805 operations; 3336.37 current ops/sec; [INSERT AverageLatency(us)=3127.49] [READ AverageLatency(us)=1451498.25]
2014-08-14 01:17:00:492 4591 sec: 19667517 operations; 11670.03 current ops/sec; [INSERT AverageLatency(us)=1857.15] [READ AverageLatency(us)=44675.19]
2014-08-14 01:17:10:493 4601 sec: 19773429 operations; 10590.14 current ops/sec; [INSERT AverageLatency(us)=1403.17] [READ AverageLatency(us)=87010.42]
2014-08-14 01:17:20:494 4611 sec: 19831569 operations; 5813.42 current ops/sec; [INSERT AverageLatency(us)=230.71] [READ AverageLatency(us)=28544.22]
2014-08-14 01:17:30:495 4621 sec: 19831569 operations; 0 current ops/sec;
2014-08-14 01:17:40:496 4631 sec: 19856103 operations; 2453.15 current ops/sec; [INSERT AverageLatency(us)=11] [READ AverageLatency(us)=852941.94]
2014-08-14 01:17:50:497 4641 sec: 20006528 operations; 15041 current ops/sec; [INSERT AverageLatency(us)=2796.15] [READ AverageLatency(us)=60604.31]
2014-08-14 01:18:00:498 4651 sec: 20115452 operations; 10891.31 current ops/sec; [INSERT AverageLatency(us)=1190.2] [READ AverageLatency(us)=56315.63]
2014-08-14 01:18:10:606 4661 sec: 20256305 operations; 13934.8 current ops/sec; [INSERT AverageLatency(us)=1041.29] [READ AverageLatency(us)=67958.28]
2014-08-14 01:18:20:606 4671 sec: 20300569 operations; 4426.4 current ops/sec; [INSERT AverageLatency(us)=468.99] [READ AverageLatency(us)=61644.09]
2014-08-14 01:18:30:607 4681 sec: 20300569 operations; 0 current ops/sec;
2014-08-14 01:18:40:608 4691 sec: 20342568 operations; 4199.48 current ops/sec; [INSERT AverageLatency(us)=3115.87] [READ AverageLatency(us)=886029.54]
2014-08-14 01:18:50:609 4701 sec: 20506102 operations; 16351.76 current ops/sec; [INSERT AverageLatency(us)=535.51] [READ AverageLatency(us)=67527.19]
2014-08-14 01:19:00:610 4711 sec: 20545024 operations; 3891.81 current ops/sec; [INSERT AverageLatency(us)=3005.51] [READ AverageLatency(us)=50405.2]
2014-08-14 01:19:10:611 4721 sec: 20545024 operations; 0 current ops/sec;
2014-08-14 01:19:20:612 4731 sec: 20545024 operations; 0 current ops/sec;
2014-08-14 01:19:30:613 4741 sec: 20545024 operations; 0 current ops/sec;
2014-08-14 01:19:40:614 4751 sec: 20644365 operations; 9933.11 current ops/sec; [INSERT AverageLatency(us)=1220.82] [READ AverageLatency(us)=715283.24]
2014-08-14 01:19:50:615 4761 sec: 20786006 operations; 14162.68 current ops/sec; [INSERT AverageLatency(us)=486.1] [READ AverageLatency(us)=77546.43]
2014-08-14 01:20:00:616 4771 sec: 20923556 operations; 13753.62 current ops/sec; [INSERT AverageLatency(us)=1254.96] [READ AverageLatency(us)=46651.7]
2014-08-14 01:20:10:618 4781 sec: 21056386 operations; 13280.34 current ops/sec; [INSERT AverageLatency(us)=789.1] [READ AverageLatency(us)=50512.1]
2014-08-14 01:20:20:619 4791 sec: 21057863 operations; 147.69 current ops/sec; [INSERT AverageLatency(us)=37592.61] [READ AverageLatency(us)=19213.5]
2014-08-14 01:20:30:620 4801 sec: 21060225 operations; 236.18 current ops/sec; [INSERT AverageLatency(us)=64819.98] [READ AverageLatency(us)=16800893.38]
2014-08-14 01:20:40:621 4811 sec: 21202893 operations; 14265.37 current ops/sec; [INSERT AverageLatency(us)=1112.51] [READ AverageLatency(us)=109524.38]
2014-08-14 01:20:50:622 4821 sec: 21232389 operations; 2949.31 current ops/sec; [INSERT AverageLatency(us)=64.19] [READ AverageLatency(us)=84778.95]
2014-08-14 01:21:00:624 4831 sec: 21279318 operations; 4691.96 current ops/sec; [INSERT AverageLatency(us)=19.23] [READ AverageLatency(us)=592179.8]
2014-08-14 01:21:10:625 4841 sec: 21388065 operations; 10873.61 current ops/sec; [INSERT AverageLatency(us)=1799.81] [READ AverageLatency(us)=73358.6]
2014-08-14 01:21:20:626 4851 sec: 21473881 operations; 8580.74 current ops/sec; [INSERT AverageLatency(us)=473.37] [READ AverageLatency(us)=53451.82]
2014-08-14 01:21:30:627 4861 sec: 21578229 operations; 10433.76 current ops/sec; [INSERT AverageLatency(us)=2195.07] [READ AverageLatency(us)=68296.74]
2014-08-14 01:21:40:629 4871 sec: 21630392 operations; 5215.26 current ops/sec; [INSERT AverageLatency(us)=1354.53] [READ AverageLatency(us)=41584.11]