Typhoeus::Hydra

    # File lib/typhoeus/hydra.rb, line 18
18:     def self.hydra
19:       @hydra ||= new
20:     end

    # File lib/typhoeus/hydra.rb, line 22
22:     def self.hydra=(val)
23:       @hydra = val
24:     end

    # File lib/typhoeus/hydra.rb, line 3
 3:     def initialize(options = {})
 4:       @memoize_requests = true
 5:       @multi       = Multi.new
 6:       @easy_pool   = []
 7:       initial_pool_size = options[:initial_pool_size] || 10
 8:       @max_concurrency = options[:max_concurrency] || 200
 9:       initial_pool_size.times { @easy_pool << Easy.new }
10:       @stubs       = []
11:       @memoized_requests = {}
12:       @retrieved_from_cache = {}
13:       @queued_requests = []
14:       @running_requests = 0
15:       @stubbed_request_count = 0
16:     end

    # File lib/typhoeus/hydra.rb, line 84
84:     def cache_getter(&block)
85:       @cache_getter = block
86:     end

    # File lib/typhoeus/hydra.rb, line 88
88:     def cache_setter(&block)
89:       @cache_setter = block
90:     end

    # File lib/typhoeus/hydra.rb, line 26
26:     def clear_cache_callbacks
27:       @cache_setter = nil
28:       @cache_getter = nil
29:     end

    # File lib/typhoeus/hydra.rb, line 31
31:     def clear_stubs
32:       @stubs = []
33:     end

    # File lib/typhoeus/hydra.rb, line 80
80:     def disable_memoization
81:       @memoize_requests = false
82:     end

    # File lib/typhoeus/hydra.rb, line 35
35:     def fire_and_forget
36:       @queued_requests.each {|r| queue(r, false)}
37:       @multi.fire_and_forget
38:     end

    # File lib/typhoeus/hydra.rb, line 92
92:     def on_complete(&block)
93:       @on_complete = block
94:     end

    # File lib/typhoeus/hydra.rb, line 96
96:     def on_complete=(proc)
97:       @on_complete = proc
98:     end

    # File lib/typhoeus/hydra.rb, line 40
40:     def queue(request, obey_concurrency_limit = true)
41:       return if assign_to_stub(request)
42: 
43:       if @running_requests >= @max_concurrency && obey_concurrency_limit
44:         @queued_requests << request
45:       else
46:         if request.method == :get
47:           if @memoize_requests && @memoized_requests.has_key?(request.url)
48:             if response = @retrieved_from_cache[request.url]
49:               request.response = response
50:               request.call_handlers
51:             else
52:               @memoized_requests[request.url] << request
53:             end
54:           else
55:             @memoized_requests[request.url] = [] if @memoize_requests
56:             get_from_cache_or_queue(request)
57:           end
58:         else
59:           get_from_cache_or_queue(request)
60:         end
61:       end
62:     end

    # File lib/typhoeus/hydra.rb, line 64
64:     def run
65:       while @stubbed_request_count > 0
66:         @stubs.each do |m|
67:           while request = m.requests.shift
68:             @stubbed_request_count -= 1
69:             m.response.request = request
70:             handle_request(request, m.response)
71:           end
72:         end
73:       end
74: 
75:       @multi.perform
76:       @memoized_requests = {}
77:       @retrieved_from_cache = {}
78:     end

     # File lib/typhoeus/hydra.rb, line 100
100:     def stub(method, url)
101:       @stubs << HydraMock.new(url, method)
102:       @stubs.last
103:     end

     # File lib/typhoeus/hydra.rb, line 105
105:     def assign_to_stub(request)
106:       m = @stubs.detect {|stub| stub.matches? request}
107:       if m
108:         m.add_request(request)
109:         @stubbed_request_count += 1
110:       else
111:         nil
112:       end
113:     end

     # File lib/typhoeus/hydra.rb, line 131
131:     def get_easy_object(request)
132:       @running_requests += 1
133: 
134:       easy = @easy_pool.pop || Easy.new
135:       easy.verbose          = request.verbose
136:       if request.username || request.password
137:         auth = { :username => request.username, :password => request.password }
138:         auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.auth_method.to_s.upcase}".to_sym] if request.auth_method
139:         easy.auth = auth
140:       end
141:       easy.url          = request.url
142:       easy.method       = request.method
143:       easy.params       = request.params  if request.method == :post && !request.params.nil?
144:       easy.headers      = request.headers if request.headers
145:       easy.request_body = request.body    if request.body
146:       easy.timeout      = request.timeout if request.timeout
147:       easy.connect_timeout = request.connect_timeout if request.connect_timeout
148:       easy.follow_location = request.follow_location if request.follow_location
149:       easy.max_redirects = request.max_redirects if request.max_redirects
150:       easy.proxy = request.proxy if request.proxy
151:       easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
152:       easy.ssl_cert         = request.ssl_cert
153:       easy.ssl_cert_type    = request.ssl_cert_type
154:       easy.ssl_key          = request.ssl_key
155:       easy.ssl_key_type     = request.ssl_key_type
156:       easy.ssl_key_password = request.ssl_key_password
157:       easy.ssl_cacert       = request.ssl_cacert
158:       easy.ssl_capath       = request.ssl_capath
159:       easy.verbose          = request.verbose
160: 
161:       easy.on_success do |easy|
162:         queue_next
163:         handle_request(request, response_from_easy(easy, request))
164:         release_easy_object(easy)
165:       end
166:       easy.on_failure do |easy|
167:         queue_next
168:         handle_request(request, response_from_easy(easy, request))
169:         release_easy_object(easy)
170:       end
171:       easy.set_headers
172:       easy
173:     end

     # File lib/typhoeus/hydra.rb, line 116
116:     def get_from_cache_or_queue(request)
117:       if @cache_getter
118:         val = @cache_getter.call(request)
119:         if val
120:           @retrieved_from_cache[request.url] = val
121:           handle_request(request, val, false)
122:         else
123:           @multi.add(get_easy_object(request))
124:         end
125:       else
126:         @multi.add(get_easy_object(request))
127:       end
128:     end

     # File lib/typhoeus/hydra.rb, line 188
188:     def handle_request(request, response, live_request = true)
189:       request.response = response
190: 
191:       if live_request && request.cache_timeout && @cache_setter
192:         @cache_setter.call(request)
193:       end
194:       @on_complete.call(response) if @on_complete
195: 
196:       request.call_handlers
197:       if requests = @memoized_requests[request.url]
198:         requests.each do |r|
199:           r.response = response
200:           r.call_handlers
201:         end
202:       end
203:     end

     # File lib/typhoeus/hydra.rb, line 176
176:     def queue_next
177:       @running_requests -= 1
178:       queue(@queued_requests.pop) unless @queued_requests.empty?
179:     end

     # File lib/typhoeus/hydra.rb, line 182
182:     def release_easy_object(easy)
183:       easy.reset
184:       @easy_pool.push easy
185:     end

     # File lib/typhoeus/hydra.rb, line 206
206:     def response_from_easy(easy, request)
207:       Response.new(:code    => easy.response_code,
208:                    :headers => easy.response_header,
209:                    :body    => easy.response_body,
210:                    :time    => easy.total_time_taken,
211:                    :effective_url => easy.effective_url,
212:                    :request => request)
213:     end