Kouhei Sutou
null+****@clear*****
Wed Jul 22 13:12:33 JST 2015
Kouhei Sutou 2015-07-22 13:12:33 +0900 (Wed, 22 Jul 2015) New Revision: ca90830890c486dc055dac94de655d626fd503ab https://github.com/groonga/groonga/commit/ca90830890c486dc055dac94de655d626fd503ab Message: sharding: don't open shard table until it's needed Modified files: plugins/sharding/logical_count.rb plugins/sharding/logical_enumerator.rb plugins/sharding/logical_range_filter.rb plugins/sharding/logical_select.rb plugins/sharding/logical_table_remove.rb Modified: plugins/sharding/logical_count.rb (+16 -10) =================================================================== --- plugins/sharding/logical_count.rb 2015-07-22 13:12:11 +0900 (a2582dc) +++ plugins/sharding/logical_count.rb 2015-07-22 13:12:33 +0900 (bea79dc) @@ -17,23 +17,22 @@ module Groonga filter = input[:filter] total = 0 - enumerator.each do |table, shard_key, shard_range| - total += count_n_records(table, filter, - shard_key, shard_range, + enumerator.each do |shard, shard_range| + total += count_n_records(filter, shard, shard_range, enumerator.target_range) end writer.write(total) end private - def log_use_range_index(use, table, line, method) + def log_use_range_index(use, table_name, line, method) message = "[logical_count]" if use message << "[range-index]" else message << "[select]" end - message << " <#{table.name}>" + message << " <#{table_name}>" Context.instance.logger.log(Logger::Level::DEBUG, __FILE__, line, @@ -41,17 +40,24 @@ module Groonga message) end - def count_n_records(table, filter, - shard_key, shard_range, - target_range) + def count_n_records(filter, shard, shard_range, target_range) cover_type = target_range.cover_type(shard_range) return 0 if cover_type == :none + shard_key = shard.key + if shard_key.nil? + message = "[logical_count] shard_key doesn't exist: " + + "<#{shard.key_name}>" + raise InvalidArgument, message + end + table = shard.table + table_name = shard.table_name + expression_builder = RangeExpressionBuilder.new(shard_key, target_range, filter) if cover_type == :all - log_use_range_index(false, table, __LINE__, __method__) + log_use_range_index(false, table_name, __LINE__, __method__) if filter.nil? return table.size else @@ -70,7 +76,7 @@ module Groonga end use_range_index = (!range_index.nil?) - log_use_range_index(use_range_index, table, __LINE__, __method__) + log_use_range_index(use_range_index, table_name, __LINE__, __method__) case cover_type when :partial_min Modified: plugins/sharding/logical_enumerator.rb (+22 -19) =================================================================== --- plugins/sharding/logical_enumerator.rb 2015-07-22 13:12:11 +0900 (432bcea) +++ plugins/sharding/logical_enumerator.rb 2015-07-22 13:12:33 +0900 (c34b1a6) @@ -22,7 +22,6 @@ module Groonga def each_internal(order) context = Context.instance each_shard_with_around(order) do |prev_shard, current_shard, next_shard| - table = current_shard.table shard_range_data = current_shard.range_data shard_range = nil @@ -52,16 +51,7 @@ module Groonga shard_range_data.day) end - physical_shard_key_name = "#{table.name}.#{@shard_key_name}" - shard_key = context[physical_shard_key_name] - if shard_key.nil? - message = - "[#{@command_name}] shard_key doesn't exist: " + - "<#{physical_shard_key_name}>" - raise InvalidArgument, message - end - - yield(table, shard_key, shard_range) + yield(current_shard, shard_range) end end @@ -70,10 +60,10 @@ module Groonga prefix = "#{@logical_table}_" shards = [nil] - context.database.each_table(:prefix => prefix, - :order_by => :key, - :order => order) do |table| - shard_range_raw = table.name[prefix.size..-1] + context.database.each_name(:prefix => prefix, + :order_by => :key, + :order => order) do |name| + shard_range_raw = name[prefix.size..-1] case shard_range_raw when /\A(\d{4})(\d{2})\z/ @@ -84,7 +74,7 @@ module Groonga next end - shards << Shard.new(table, shard_range_data) + shards << Shard.new(name, @shard_key_name, shard_range_data) next if shards.size < 3 yield(*shards) shards.shift @@ -119,11 +109,24 @@ module Groonga end class Shard - attr_reader :table, :range_data - def initialize(table, range_data) - @table = table + attr_reader :table_name, :key_name, :range_data + def initialize(table_name, key_name, range_data) + @table_name = table_name + @key_name = key_name @range_data = range_data end + + def table + @table ||= Context.instance[@table_name] + end + + def full_key_name + "#{@table_name}.#{@key_name}" + end + + def key + @key ||= Context.instance[full_key_name] + end end class ShardRangeData Modified: plugins/sharding/logical_range_filter.rb (+54 -44) =================================================================== --- plugins/sharding/logical_range_filter.rb 2015-07-22 13:12:11 +0900 (90c927e) +++ plugins/sharding/logical_range_filter.rb 2015-07-22 13:12:33 +0900 (fca7a37) @@ -155,23 +155,21 @@ module Groonga end def execute - first_table = nil + first_shard = nil enumerator =****@conte***** + target_range = enumerator.target_range if****@conte***** == :descending each_method = :reverse_each else each_method = :each end - enumerator.send(each_method) do |table, shard_key, shard_range| - first_table ||= table - next if table.empty? - - shard_executor = ShardExecutor.new(@context, - table, shard_key, shard_range) + enumerator.send(each_method) do |shard, shard_range| + first_shard ||= shard + shard_executor = ShardExecutor.new(@context, shard, shard_range) shard_executor.execute break if****@conte*****_limit == 0 end - if first_table.nil? + if first_shard.nil? message = "[logical_range_filter] no shard exists: " + "logical_table: <#{enumerator.logical_table}>: " + @@ -180,17 +178,16 @@ module Groonga end if****@conte*****_sets.empty? result_set = HashTable.create(:flags => ObjectFlags::WITH_SUBREC, - :key_type => first_table) + :key_type => first_shard.table) @context.result_sets << result_set end end end class ShardExecutor - def initialize(context, table, shard_key, shard_range) + def initialize(context, shard, shard_range) @context = context - @table = table - @shard_key = shard_key + @shard = shard @shard_range = shard_range @filter =****@conte***** @@ -200,26 +197,36 @@ module Groonga @target_range =****@conte*****_range @cover_type = @target_range.cover_type(@shard_range) - - @expression_builder = RangeExpressionBuilder.new(@shard_key, - @target_range, - @filter) end def execute return if @cover_type == :none + return if****@shard*****? - index_info = @shard_key.find_index(Operator::LESS) + shard_key =****@shard***** + if shard_key.nil? + message = "[logical_range_filter] shard_key doesn't exist: " + + "<#{@shard.key_name}>" + raise InvalidArgument, message + end + + expression_builder = RangeExpressionBuilder.new(shard_key, + @target_range, + @filter) + + index_info = shard_key.find_index(Operator::LESS) if index_info range_index = index_info.index - range_index = nil unless use_range_index?(range_index) + unless use_range_index?(range_index, expression_builder) + range_index = nil + end else range_index = nil end case @cover_type when :all - filter_shard_all(range_index) + filter_shard_all(range_index, expression_builder) when :partial_min if range_index filter_by_range(range_index, @@ -227,7 +234,7 @@ module Groonga nil, nil) else filter_table do |expression| - @expression_builder.build_partial_min(expression) + expression_builder.build_partial_min(expression) end end when :partial_max @@ -237,7 +244,7 @@ module Groonga @target_range.max, @target_range.max_border) else filter_table do |expression| - @expression_builder.build_partial_max(expression) + expression_builder.build_partial_max(expression) end end when :partial_min_and_max @@ -247,7 +254,7 @@ module Groonga @target_range.max, @target_range.max_border) else filter_table do |expression| - @expression_builder.build_partial_min_and_max(expression) + expression_builder.build_partial_min_and_max(expression) end end end @@ -261,7 +268,7 @@ module Groonga else message << "[select] " end - message << "<#{@table.name}>: " + message << "<#{@shard.table_name}>: " message << reason Context.instance.logger.log(Logger::Level::DEBUG, __FILE__, @@ -272,7 +279,7 @@ module Groonga use end - def use_range_index?(range_index) + def use_range_index?(range_index, expression_builder) case****@conte*****_range_index when true return decide_use_range_index(true, @@ -292,7 +299,7 @@ module Groonga end required_n_records =****@conte*****_offset + current_limit - max_n_records =****@table***** + max_n_records =****@shard***** if max_n_records <= required_n_records reason = "the number of required records (#{required_n_records}) " reason << ">= " @@ -313,47 +320,48 @@ module Groonga __LINE__, __method__) end + table =****@shard***** estimated_n_records = 0 case @cover_type when :all if @filter - create_expression(@table) do |expression| - @expression_builder.build_all(expression) + create_expression(table) do |expression| + expression_builder.build_all(expression) unless range_index_available_expression?(expression, __LINE__, __method__) return false end - estimated_n_records = expression.estimate_size(@table) + estimated_n_records = expression.estimate_size(table) end else estimated_n_records = max_n_records end when :partial_min - create_expression(@table) do |expression| - @expression_builder.build_partial_min(expression) + create_expression(table) do |expression| + expression_builder.build_partial_min(expression) unless range_index_available_expression?(expression, __LINE__, __method__) return false end - estimated_n_records = expression.estimate_size(@table) + estimated_n_records = expression.estimate_size(table) end when :partial_max - create_expression(@table) do |expression| - @expression_builder.build_partial_max(expression) + create_expression(table) do |expression| + expression_builder.build_partial_max(expression) unless range_index_available_expression?(expression, __LINE__, __method__) return false end - estimated_n_records = expression.estimate_size(@table) + estimated_n_records = expression.estimate_size(table) end when :partial_min_and_max - create_expression(@table) do |expression| - @expression_builder.build_partial_min_and_max(expression) + create_expression(table) do |expression| + expression_builder.build_partial_min_and_max(expression) unless range_index_available_expression?(expression, __LINE__, __method__) return false end - estimated_n_records = expression.estimate_size(@table) + estimated_n_records = expression.estimate_size(table) end end @@ -421,10 +429,11 @@ module Groonga nil end - def filter_shard_all(range_index) + def filter_shard_all(range_index, expression_builder) + table =****@shard***** if****@filte*****? - if****@table***** <=****@conte*****_offset - @context.current_offset -=****@table***** + if table.size <=****@conte*****_offset + @context.current_offset -= table.size return end if range_index @@ -432,7 +441,7 @@ module Groonga nil, nil, nil, nil) else - sort_result_set(@table) + sort_result_set(table) end else if range_index @@ -441,7 +450,7 @@ module Groonga nil, nil) else filter_table do |expression| - @expression_builder.build_all(expression) + expression_builder.build_all(expression) end end end @@ -537,9 +546,10 @@ module Groonga end def filter_table - create_expression(@table) do |expression| + table =****@shard***** + create_expression(table) do |expression| yield(expression) - result_set =****@table*****(expression) + result_set = table.select(expression) sort_result_set(result_set) end end Modified: plugins/sharding/logical_select.rb (+29 -25) =================================================================== --- plugins/sharding/logical_select.rb 2015-07-22 13:12:11 +0900 (3261a89) +++ plugins/sharding/logical_select.rb 2015-07-22 13:12:33 +0900 (3761ecd) @@ -412,17 +412,14 @@ module Groonga private def execute_search - first_table = nil + first_shard = nil enumerator =****@conte***** - enumerator.each do |table, shard_key, shard_range| - first_table ||= table - next if table.empty? - - shard_executor = ShardExecutor.new(@context, - table, shard_key, shard_range) + enumerator.each do |shard, shard_range| + first_shard ||= shard + shard_executor = ShardExecutor.new(@context, shard, shard_range) shard_executor.execute end - if first_table.nil? + if first_shard.nil? message = "[logical_select] no shard exists: " + "logical_table: <#{enumerator.logical_table}>: " + @@ -431,7 +428,7 @@ module Groonga end if****@conte*****_sets.empty? result_set = HashTable.create(:flags => ObjectFlags::WITH_SUBREC, - :key_type => first_table) + :key_type => first_shard.table) @context.result_sets << result_set end end @@ -511,10 +508,9 @@ module Groonga end class ShardExecutor - def initialize(context, table, shard_key, shard_range) + def initialize(context, shard, shard_range) @context = context - @table = table - @shard_key = shard_key + @shard = shard @shard_range = shard_range @filter =****@conte***** @@ -525,40 +521,47 @@ module Groonga @target_range =****@conte*****_range @cover_type = @target_range.cover_type(@shard_range) - - @expression_builder = RangeExpressionBuilder.new(@shard_key, - @target_range, - @filter) end def execute return if @cover_type == :none + return if****@shard*****? + + shard_key =****@shard***** + if shard_key.nil? + message = "[logical_select] shard_key doesn't exist: " + + "<#{@shard.key_name}>" + raise InvalidArgument, message + end + expression_builder = RangeExpressionBuilder.new(shard_key, + @target_range, + @filter) case @cover_type when :all - filter_shard_all + filter_shard_all(expression_builder) when :partial_min filter_table do |expression| - @expression_builder.build_partial_min(expression) + expression_builder.build_partial_min(expression) end when :partial_max filter_table do |expression| - @expression_builder.build_partial_max(expression) + expression_builder.build_partial_max(expression) end when :partial_min_and_max filter_table do |expression| - @expression_builder.build_partial_min_and_max(expression) + expression_builder.build_partial_min_and_max(expression) end end end private - def filter_shard_all + def filter_shard_all(expression_builder) if****@filte*****? - add_result_set(@table) + add_result_set(@shard.table) else filter_table do |expression| - @expression_builder.build_all(expression) + expression_builder.build_all(expression) end end end @@ -573,9 +576,10 @@ module Groonga end def filter_table - create_expression(@table) do |expression| + table =****@shard***** + create_expression(table) do |expression| yield(expression) - add_result_set(@table.select(expression)) + add_result_set(table.select(expression)) end end Modified: plugins/sharding/logical_table_remove.rb (+11 -6) =================================================================== --- plugins/sharding/logical_table_remove.rb 2015-07-22 13:12:11 +0900 (f5c31c7) +++ plugins/sharding/logical_table_remove.rb 2015-07-22 13:12:33 +0900 (6e988f6) @@ -15,20 +15,25 @@ module Groonga enumerator = LogicalEnumerator.new("logical_table_remove", input) succeess = true - enumerator.each do |table, shard_key, shard_range| - remove_table(table, - shard_key, - shard_range, - enumerator.target_range) + enumerator.each do |shard, shard_range| + remove_table(shard, shard_range, enumerator.target_range) end writer.write(succeess) end private - def remove_table(table, shard_key, shard_range, target_range) + def remove_table(shard, shard_range, target_range) cover_type = target_range.cover_type(shard_range) return if cover_type == :none + shard_key = shard.key + if shard_key.nil? + message = "[logical_table_remove] shard_key doesn't exist: " + + "<#{shard.key_name}>" + raise InvalidArgument, message + end + table = shard.table + expression_builder = RangeExpressionBuilder.new(shard_key, target_range, nil) -------------- next part -------------- HTML����������������������������...Descargar
