From 114af0b4ac8784eb781ba0cf295a4ae7a5bfadef Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 5 Jan 2018 17:46:19 +0530 Subject: [PATCH 1/9] Use a custom 'Message' class --- lib/segment/analytics/message.rb | 17 +++++++++++++++++ lib/segment/analytics/worker.rb | 6 +++--- 2 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 lib/segment/analytics/message.rb diff --git a/lib/segment/analytics/message.rb b/lib/segment/analytics/message.rb new file mode 100644 index 0000000..47b47b0 --- /dev/null +++ b/lib/segment/analytics/message.rb @@ -0,0 +1,17 @@ +require 'forwardable' + +module Segment + class Analytics + # Represents a message to be sent to the API + class Message + extend Forwardable + + def initialize(hash) + @hash = hash + end + + def_delegators :@hash, :to_json # TODO: Cache and reuse + def_delegators :@hash, :[] + end + end +end diff --git a/lib/segment/analytics/worker.rb b/lib/segment/analytics/worker.rb index f619a1b..5e94b5f 100644 --- a/lib/segment/analytics/worker.rb +++ b/lib/segment/analytics/worker.rb @@ -1,7 +1,7 @@ require 'segment/analytics/defaults' -require 'segment/analytics/utils' -require 'segment/analytics/defaults' +require 'segment/analytics/message' require 'segment/analytics/request' +require 'segment/analytics/utils' module Segment class Analytics @@ -38,7 +38,7 @@ def run @lock.synchronize do until @batch.length >= @batch_size || @queue.empty? - @batch << @queue.pop + @batch << Message.new(@queue.pop) end end From 5174fbdec01fdaf639a59ff1dc0faccee94abee9 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 5 Jan 2018 18:03:22 +0530 Subject: [PATCH 2/9] Add custom 'MessageBatch' class --- lib/segment/analytics/message_batch.rb | 18 ++++++++++++++++++ lib/segment/analytics/worker.rb | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 lib/segment/analytics/message_batch.rb diff --git a/lib/segment/analytics/message_batch.rb b/lib/segment/analytics/message_batch.rb new file mode 100644 index 0000000..bc9c95c --- /dev/null +++ b/lib/segment/analytics/message_batch.rb @@ -0,0 +1,18 @@ +module Segment + class Analytics + # A batch of `Message`s to be sent to the API + class MessageBatch + extend Forwardable + + def initialize + @messages = [] + end + + def_delegators :@messages, :to_json # TODO: Cache and reuse + def_delegators :@messages, :<< + def_delegators :@messages, :clear + def_delegators :@messages, :empty? + def_delegators :@messages, :length + end + end +end diff --git a/lib/segment/analytics/worker.rb b/lib/segment/analytics/worker.rb index 5e94b5f..93e2067 100644 --- a/lib/segment/analytics/worker.rb +++ b/lib/segment/analytics/worker.rb @@ -1,5 +1,6 @@ require 'segment/analytics/defaults' require 'segment/analytics/message' +require 'segment/analytics/message_batch' require 'segment/analytics/request' require 'segment/analytics/utils' @@ -26,7 +27,7 @@ def initialize(queue, write_key, options = {}) @write_key = write_key @batch_size = options[:batch_size] || Queue::BATCH_SIZE @on_error = options[:on_error] || proc { |status, error| } - @batch = [] + @batch = MessageBatch.new @lock = Mutex.new end From f6e659b44be55a27d2d7ec94a8f7a61d226be425 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 5 Jan 2018 18:04:14 +0530 Subject: [PATCH 3/9] Reject messages that exceed the maximum allowed size --- lib/segment/analytics/defaults.rb | 4 ++++ lib/segment/analytics/message.rb | 6 +++++ lib/segment/analytics/message_batch.rb | 12 +++++++++- spec/segment/analytics/message_batch_spec.rb | 25 ++++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 spec/segment/analytics/message_batch_spec.rb diff --git a/lib/segment/analytics/defaults.rb b/lib/segment/analytics/defaults.rb index 550bf67..18cf861 100644 --- a/lib/segment/analytics/defaults.rb +++ b/lib/segment/analytics/defaults.rb @@ -17,6 +17,10 @@ module Queue MAX_SIZE = 10000 end + module Message + MAX_BYTES = 32768 # 32Kb + end + module BackoffPolicy MIN_TIMEOUT_MS = 100 MAX_TIMEOUT_MS = 10000 diff --git a/lib/segment/analytics/message.rb b/lib/segment/analytics/message.rb index 47b47b0..8b62f82 100644 --- a/lib/segment/analytics/message.rb +++ b/lib/segment/analytics/message.rb @@ -1,5 +1,7 @@ require 'forwardable' +require 'segment/analytics/defaults' + module Segment class Analytics # Represents a message to be sent to the API @@ -10,6 +12,10 @@ def initialize(hash) @hash = hash end + def too_big? + to_json.bytesize > Defaults::Message::MAX_BYTES + end + def_delegators :@hash, :to_json # TODO: Cache and reuse def_delegators :@hash, :[] end diff --git a/lib/segment/analytics/message_batch.rb b/lib/segment/analytics/message_batch.rb index bc9c95c..6bf77bf 100644 --- a/lib/segment/analytics/message_batch.rb +++ b/lib/segment/analytics/message_batch.rb @@ -1,15 +1,25 @@ +require 'segment/analytics/logging' + module Segment class Analytics # A batch of `Message`s to be sent to the API class MessageBatch extend Forwardable + include Segment::Analytics::Logging def initialize @messages = [] end + def <<(message) + if message.too_big? + logger.error('a message exceeded the maximum allowed size') + else + @messages << message + end + end + def_delegators :@messages, :to_json # TODO: Cache and reuse - def_delegators :@messages, :<< def_delegators :@messages, :clear def_delegators :@messages, :empty? def_delegators :@messages, :length diff --git a/spec/segment/analytics/message_batch_spec.rb b/spec/segment/analytics/message_batch_spec.rb new file mode 100644 index 0000000..e50f4d0 --- /dev/null +++ b/spec/segment/analytics/message_batch_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +module Segment + class Analytics + describe MessageBatch do + describe '#<<' do + subject { described_class.new } + + it 'appends messages' do + subject << Message.new('a' => 'b') + expect(subject.length).to eq(1) + end + + it 'rejects messages that exceed the maximum allowed size' do + max_bytes = Segment::Analytics::Defaults::Message::MAX_BYTES + hash = { 'a' => 'b' * max_bytes } + message = Message.new(hash) + + subject << message + expect(subject.length).to eq(0) + end + end + end + end +end From feedf1a345f8f5c6e5386401fba858b3af32a7a5 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 5 Jan 2018 18:24:38 +0530 Subject: [PATCH 4/9] Cache JSON generation for individual messages --- lib/segment/analytics/message.rb | 11 ++++---- lib/segment/analytics/message_batch.rb | 2 +- spec/segment/analytics/message_spec.rb | 35 ++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 spec/segment/analytics/message_spec.rb diff --git a/lib/segment/analytics/message.rb b/lib/segment/analytics/message.rb index 8b62f82..b11b121 100644 --- a/lib/segment/analytics/message.rb +++ b/lib/segment/analytics/message.rb @@ -1,13 +1,9 @@ -require 'forwardable' - require 'segment/analytics/defaults' module Segment class Analytics # Represents a message to be sent to the API class Message - extend Forwardable - def initialize(hash) @hash = hash end @@ -16,8 +12,11 @@ def too_big? to_json.bytesize > Defaults::Message::MAX_BYTES end - def_delegators :@hash, :to_json # TODO: Cache and reuse - def_delegators :@hash, :[] + # Since the hash is expected to not be modified (set at initialization), + # the JSON version can be cached after the first computation. + def to_json(*args) + @json ||= @hash.to_json(*args) + end end end end diff --git a/lib/segment/analytics/message_batch.rb b/lib/segment/analytics/message_batch.rb index 6bf77bf..0cf1dc4 100644 --- a/lib/segment/analytics/message_batch.rb +++ b/lib/segment/analytics/message_batch.rb @@ -19,7 +19,7 @@ def <<(message) end end - def_delegators :@messages, :to_json # TODO: Cache and reuse + def_delegators :@messages, :to_json def_delegators :@messages, :clear def_delegators :@messages, :empty? def_delegators :@messages, :length diff --git a/spec/segment/analytics/message_spec.rb b/spec/segment/analytics/message_spec.rb new file mode 100644 index 0000000..b156183 --- /dev/null +++ b/spec/segment/analytics/message_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +module Segment + class Analytics + describe Message do + describe '#to_json' do + it 'caches JSON conversions' do + # Keeps track of the number of times to_json was called + nested_obj = Class.new do + attr_reader :to_json_call_count + + def initialize + @to_json_call_count = 0 + end + + def to_json(*_) + @to_json_call_count += 1 + '{}' + end + end.new + + message = Message.new('some_key' => nested_obj) + expect(nested_obj.to_json_call_count).to eq(0) + + message.to_json + expect(nested_obj.to_json_call_count).to eq(1) + + # When called a second time, the call count shouldn't increase + message.to_json + expect(nested_obj.to_json_call_count).to eq(1) + end + end + end + end +end From 6682f02945050fa16a4e8d58b7bfd1c00983721c Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 9 Jan 2018 18:42:40 +0530 Subject: [PATCH 5/9] Limit batch size to 500kb --- lib/segment/analytics/defaults.rb | 6 +++- lib/segment/analytics/message.rb | 6 +++- lib/segment/analytics/message_batch.rb | 34 ++++++++++++++++++-- lib/segment/analytics/worker.rb | 7 ++-- spec/segment/analytics/message_batch_spec.rb | 2 +- spec/segment/analytics/worker_spec.rb | 7 ++-- 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/lib/segment/analytics/defaults.rb b/lib/segment/analytics/defaults.rb index 18cf861..8562346 100644 --- a/lib/segment/analytics/defaults.rb +++ b/lib/segment/analytics/defaults.rb @@ -13,7 +13,6 @@ module Request end module Queue - BATCH_SIZE = 100 MAX_SIZE = 10000 end @@ -21,6 +20,11 @@ module Message MAX_BYTES = 32768 # 32Kb end + module MessageBatch + MAX_BYTES = 512_000 # 500Kb + MAX_SIZE = 100 + end + module BackoffPolicy MIN_TIMEOUT_MS = 100 MAX_TIMEOUT_MS = 10000 diff --git a/lib/segment/analytics/message.rb b/lib/segment/analytics/message.rb index b11b121..e00a74d 100644 --- a/lib/segment/analytics/message.rb +++ b/lib/segment/analytics/message.rb @@ -9,7 +9,11 @@ def initialize(hash) end def too_big? - to_json.bytesize > Defaults::Message::MAX_BYTES + json_size > Defaults::Message::MAX_BYTES + end + + def json_size + to_json.bytesize end # Since the hash is expected to not be modified (set at initialization), diff --git a/lib/segment/analytics/message_batch.rb b/lib/segment/analytics/message_batch.rb index 0cf1dc4..bc44971 100644 --- a/lib/segment/analytics/message_batch.rb +++ b/lib/segment/analytics/message_batch.rb @@ -6,9 +6,12 @@ class Analytics class MessageBatch extend Forwardable include Segment::Analytics::Logging + include Segment::Analytics::Defaults::MessageBatch - def initialize + def initialize(max_message_count) @messages = [] + @max_message_count = max_message_count + @json_size = 0 end def <<(message) @@ -16,13 +19,40 @@ def <<(message) logger.error('a message exceeded the maximum allowed size') else @messages << message + @json_size += message.json_size + 1 # One byte for the comma end end + def full? + item_count_exhausted? || size_exhausted? + end + + def clear + @messages.clear + @json_size = 0 + end + def_delegators :@messages, :to_json - def_delegators :@messages, :clear def_delegators :@messages, :empty? def_delegators :@messages, :length + + private + + def item_count_exhausted? + @messages.length >= @max_message_count + end + + # We consider the max size here as just enough to leave room for one more + # message of the largest size possible. This is a shortcut that allows us + # to use a native Ruby `Queue` that doesn't allow peeking. The tradeoff + # here is that we might fit in less messages than possible into a batch. + # + # The alternative is to use our own `Queue` implementation that allows + # peeking, and to consider the next message size when calculating whether + # the message can be accomodated in this batch. + def size_exhausted? + @json_size >= (MAX_BYTES - Defaults::Message::MAX_BYTES) + end end end end diff --git a/lib/segment/analytics/worker.rb b/lib/segment/analytics/worker.rb index 93e2067..c57e219 100644 --- a/lib/segment/analytics/worker.rb +++ b/lib/segment/analytics/worker.rb @@ -25,9 +25,9 @@ def initialize(queue, write_key, options = {}) symbolize_keys! options @queue = queue @write_key = write_key - @batch_size = options[:batch_size] || Queue::BATCH_SIZE @on_error = options[:on_error] || proc { |status, error| } - @batch = MessageBatch.new + batch_size = options[:batch_size] || Defaults::MessageBatch::MAX_SIZE + @batch = MessageBatch.new(batch_size) @lock = Mutex.new end @@ -38,13 +38,12 @@ def run return if @queue.empty? @lock.synchronize do - until @batch.length >= @batch_size || @queue.empty? + until @batch.full? || @queue.empty? @batch << Message.new(@queue.pop) end end res = Request.new.post @write_key, @batch - @on_error.call(res.status, res.error) unless res.status == 200 @lock.synchronize { @batch.clear } diff --git a/spec/segment/analytics/message_batch_spec.rb b/spec/segment/analytics/message_batch_spec.rb index e50f4d0..850c9bd 100644 --- a/spec/segment/analytics/message_batch_spec.rb +++ b/spec/segment/analytics/message_batch_spec.rb @@ -4,7 +4,7 @@ module Segment class Analytics describe MessageBatch do describe '#<<' do - subject { described_class.new } + subject { described_class.new(100) } it 'appends messages' do subject << Message.new('a' => 'b') diff --git a/spec/segment/analytics/worker_spec.rb b/spec/segment/analytics/worker_spec.rb index 3abd4d8..7668124 100644 --- a/spec/segment/analytics/worker_spec.rb +++ b/spec/segment/analytics/worker_spec.rb @@ -6,8 +6,11 @@ class Analytics describe '#init' do it 'accepts string keys' do queue = Queue.new - worker = Segment::Analytics::Worker.new(queue, 'secret', 'batch_size' => 100) - expect(worker.instance_variable_get(:@batch_size)).to eq(100) + worker = Segment::Analytics::Worker.new(queue, + 'secret', + 'batch_size' => 100) + batch = worker.instance_variable_get(:@batch) + expect(batch.instance_variable_get(:@max_message_count)).to eq(100) end end From 39232c2135f48cb98f14628c2044c8b28a148ee7 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 9 Jan 2018 18:55:21 +0530 Subject: [PATCH 6/9] Add specs for MessageBatch#full? --- spec/segment/analytics/message_batch_spec.rb | 39 ++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/spec/segment/analytics/message_batch_spec.rb b/spec/segment/analytics/message_batch_spec.rb index 850c9bd..562196a 100644 --- a/spec/segment/analytics/message_batch_spec.rb +++ b/spec/segment/analytics/message_batch_spec.rb @@ -3,16 +3,25 @@ module Segment class Analytics describe MessageBatch do - describe '#<<' do - subject { described_class.new(100) } + subject { described_class.new(100) } + describe '#<<' do it 'appends messages' do subject << Message.new('a' => 'b') expect(subject.length).to eq(1) end it 'rejects messages that exceed the maximum allowed size' do - max_bytes = Segment::Analytics::Defaults::Message::MAX_BYTES + max_bytes = Defaults::Message::MAX_BYTES + hash = { 'a' => 'b' * max_bytes } + message = Message.new(hash) + + subject << message + expect(subject.length).to eq(0) + end + + it 'rejects messages that exceed the maximum allowed size' do + max_bytes = Defaults::Message::MAX_BYTES hash = { 'a' => 'b' * max_bytes } message = Message.new(hash) @@ -20,6 +29,30 @@ class Analytics expect(subject.length).to eq(0) end end + + describe '#full?' do + it 'returns true once item count is exceeded' do + 99.times { subject << Message.new(a: 'b') } + expect(subject.full?).to be(false) + + subject << Message.new(a: 'b') + expect(subject.full?).to be(true) + end + + it 'returns true once max size is almost exceeded' do + message = Message.new(a: 'b' * (Defaults::Message::MAX_BYTES - 10)) + + # Each message is under the individual limit + expect(message.json_size).to be < Defaults::Message::MAX_BYTES + + # Size of the batch is over the limit + expect(50 * message.json_size).to be > Defaults::MessageBatch::MAX_BYTES + + expect(subject.full?).to be(false) + 50.times { subject << message } + expect(subject.full?).to be(true) + end + end end end end From c8cf2f10cdceaf7651fb02b7f2179b25d9e7c2e7 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Tue, 9 Jan 2018 19:03:24 +0530 Subject: [PATCH 7/9] Increase number of runscope messages checked for e2e tests Since builds run in parallel, the e2e tests sometimes fail because their runscope messages are pushed down by other builds. We currently test against 7 ruby versions, and a maximum of 2 builds are triggered per PR (push + PR build). Checking 20 items should be sufficient to handle this workload. --- spec/helpers/runscope_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/helpers/runscope_client.rb b/spec/helpers/runscope_client.rb index 7d0e7f8..a68be26 100644 --- a/spec/helpers/runscope_client.rb +++ b/spec/helpers/runscope_client.rb @@ -9,7 +9,7 @@ def initialize(api_token) def requests(bucket_key) with_retries(3) do - response = @conn.get("/buckets/#{bucket_key}/messages", count: 10) + response = @conn.get("/buckets/#{bucket_key}/messages", count: 20) raise "Runscope error. #{response.body}" unless response.status == 200 From 1fcc9583383bde7ffdde09b3f95e30a0c9ca2720 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Wed, 10 Jan 2018 13:53:50 +0530 Subject: [PATCH 8/9] Fix long lines in worker_spec.rb --- spec/segment/analytics/worker_spec.rb | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/spec/segment/analytics/worker_spec.rb b/spec/segment/analytics/worker_spec.rb index 7668124..c85861a 100644 --- a/spec/segment/analytics/worker_spec.rb +++ b/spec/segment/analytics/worker_spec.rb @@ -38,8 +38,11 @@ class Analytics end.to_not raise_error end - it 'executes the error handler, before the request phase ends, if the request is invalid' do - Segment::Analytics::Request.any_instance.stub(:post).and_return(Segment::Analytics::Response.new(400, 'Some error')) + it 'executes the error handler if the request is invalid' do + Segment::Analytics::Request + .any_instance + .stub(:post) + .and_return(Segment::Analytics::Response.new(400, 'Some error')) status = error = nil on_error = proc do |yielded_status, yielded_error| @@ -49,9 +52,10 @@ class Analytics queue = Queue.new queue << {} - worker = Segment::Analytics::Worker.new queue, 'secret', :on_error => on_error + worker = described_class.new(queue, 'secret', :on_error => on_error) - # This is to ensure that Client#flush doesn't finish before calling the error handler. + # This is to ensure that Client#flush doesn't finish before calling + # the error handler. Thread.new { worker.run } sleep 0.1 # First give thread time to spin-up. sleep 0.01 while worker.is_requesting? @@ -72,7 +76,9 @@ class Analytics queue = Queue.new queue << Requested::TRACK - worker = Segment::Analytics::Worker.new queue, 'testsecret', :on_error => on_error + worker = described_class.new(queue, + 'testsecret', + :on_error => on_error) worker.run expect(queue).to be_empty From c76f94a39d66dfaad8965ac2b1e3f5a50a5fbdfc Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Wed, 10 Jan 2018 14:00:43 +0530 Subject: [PATCH 9/9] Fix indeterminate test in worker_spec.rb The assertion in the thread wouldn't trigger if the test exited first. --- spec/segment/analytics/worker_spec.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/spec/segment/analytics/worker_spec.rb b/spec/segment/analytics/worker_spec.rb index c85861a..e66db64 100644 --- a/spec/segment/analytics/worker_spec.rb +++ b/spec/segment/analytics/worker_spec.rb @@ -98,12 +98,11 @@ class Analytics queue << Requested::TRACK worker = Segment::Analytics::Worker.new(queue, 'testsecret') - Thread.new do - worker.run - expect(worker.is_requesting?).to eq(false) - end - + worker_thread = Thread.new { worker.run } eventually { expect(worker.is_requesting?).to eq(true) } + + worker_thread.join + expect(worker.is_requesting?).to eq(false) end end end