ncalls  tottime  percall  cumtime  percall filename:lineno(function)
    234/1    0.013    0.000 2399.420 2399.420 {built-in method exec}
        1    0.000    0.000 2399.420 2399.420 ./webspider/main.py:3(<module>)
        1    0.000    0.000 2399.230 2399.230 ./webspider/main.py:20(main)
        1   16.962   16.962 2399.226 2399.226 /site-packages/grab/spider/base.py:770(run)
   908957    9.572    0.000 1425.586    0.002 /site-packages/grab/spider/transport/multicurl.py:67(process_handlers)
   373146 1355.979    0.004 1355.979    0.004 {built-in method select}
   908950    2.220    0.000  561.922    0.001 /site-packages/grab/spider/transport/multicurl.py:85(iterate_results)
     3446  555.867    0.161  555.867    0.161 {method 'remove_handle' of 'pycurl.CurlMulti' objects}
     3446    0.069    0.000  349.299    0.101 /site-packages/grab/spider/base.py:645(process_network_result)
     3446    0.082    0.000  349.049    0.101 /site-packages/grab/spider/base.py:539(execute_task_handler)
     2204    1.000    0.000  348.662    0.158 ./webspider/web_spider/spider.py:112(task_initial)
   178521    2.504    0.000  186.556    0.001 ./webspider/web_spider/dao_v2.py:42(insert_new_url)
   345955    4.066    0.000  184.064    0.001 /site-packages/pymongo/collection.py:1928(update)
   345955    5.788    0.000  140.383    0.000 /site-packages/pymongo/collection.py:511(_update)
   345955   13.272    0.000  119.636    0.000 {built-in method _do_batched_write_command}
   530116    2.581    0.000  119.383    0.000 /site-packages/pymongo/network.py:58(receive_message)
   530107    0.646    0.000  117.131    0.000 /site-packages/pymongo/pool.py:208(receive_message)
  1060232    3.193    0.000  116.297    0.000 /site-packages/pymongo/network.py:73(_receive_data_on_socket)
  1060232  112.842    0.000  112.842    0.000 {method 'recv' of '_socket.socket' objects}
   345955    1.643    0.000  102.505    0.000 /site-packages/pymongo/pool.py:239(write_command)
   194511    0.518    0.000   80.746    0.000 ./webspider/web_spider/spider.py:126(<lambda>)
   184143    0.988    0.000   72.350    0.000 ./webspider/web_spider/dao_v2.py:193(exists_url)
   184144    1.309    0.000   71.302    0.000 /site-packages/pymongo/collection.py:762(find_one)
   184144    0.514    0.000   63.505    0.000 /site-packages/pymongo/cursor.py:978(next)
   184144    1.252    0.000   62.883    0.000 /site-packages/pymongo/cursor.py:884(_refresh)
   184144    1.930    0.000   60.871    0.000 /site-packages/pymongo/cursor.py:792(__send_message)
   908956   57.808    0.000   58.187    0.000 {method 'perform' of 'pycurl.CurlMulti' objects}
   184152    1.734    0.000   57.260    0.000 /site-packages/pymongo/mongo_client.py:704(_send_message_with_response)
     2204    0.022    0.000   53.740    0.024 ./webspider/web_spider/web_engine.py:165(detect_cms)
     7699    0.042    0.000   53.698    0.007 ./webspider/web_spider/web_engine.py:50(_check_cms)
4998816/3246620    3.097    0.000   50.191    0.000 {built-in method next}
   184152    0.237    0.000   48.112    0.000 /site-packages/pymongo/mongo_client.py:746(_reset_on_error)
   184152    2.495    0.000   47.875    0.000 /site-packages/pymongo/server.py:66(send_message_with_response)
     6681    0.392    0.000   41.529    0.006 ./webspider/web_spider/web_engine.py:60(_check_inner_urls)
     6681    0.874    0.000   41.136    0.006 ./webspider/web_spider/validator.py:80(get_links)
  1102858    0.979    0.000   39.656    0.000 /site-packages/grab/base.py:645(make_url_absolute)
  1102858    3.789    0.000   38.676    0.000 /python3.4/urllib/parse.py:405(urljoin)
  2787547    7.040    0.000   35.346    0.000 /python3.4/urllib/parse.py:286(urlparse)
   691926    1.931    0.000   30.790    0.000 /site-packages/pymongo/mongo_client.py:661(_get_socket)
   839681    0.660    0.000   30.737    0.000 ./webspider/web_spider/validator.py:83(<genexpr>)