diff --git a/.env b/.env new file mode 100644 index 0000000..700e857 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +DATABASE_URL='postgis://postgres:postgres@localhost:5577' diff --git a/.gitignore b/.gitignore index d87d4be..d89ea95 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ spec/reports test/tmp test/version_tmp tmp +spec/dummy/log \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 3576145..5b017e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## Latest 2022-03-16 + +* Big refactoring including: + * Support for mac address + * Fix issue with `time` type + * Fix issue with negative numerics + * Remove support for `send_at_once`, it is now synchronous by default + * Use `copy_data` block from pg + * Use `sync_put_copy_data` from pg and flush at the end + * Add support for `geometry` and `geogrphy` + * Add integrations tests using a Dummy rails App (See [spec/dummy](spec/dummy)) + * Remove the unused `use_tempfile` options from `EncodeForCopy` + * Add explicit dependency to the [pg](https://rubygems.org/gems/pg/versions/1.3.4) gem, with a version minimum to 1.3.0 as we are using the `sync_put_copy_data` method + ## 1.1.0 2018-05-24 * Add support for range data types diff --git a/README.md b/README.md index c9780af..84973e8 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,20 @@ MyModel.find_by(field_1: 'abc') ## Authors * [Lukas Fittl](https://github.com/lfittl) +* [Xavier Delamotte](https://github.com/x4d3) Credits to [Pete Brumm](https://github.com/pbrumm) who wrote pg_data_encoder and which this library repurposes. +## Developing + +``` +./start-local-db.sh +cd spec/dummy +bundle install +DATABASE_URL='postgis://postgres:postgres@localhost:5577' rake db:create db:migrate db:test:prepare +``` + ## LICENSE Copyright (c) 2018, Lukas Fittl
diff --git a/activerecord-copy.gemspec b/activerecord-copy.gemspec index 6f25cee..57a1787 100644 --- a/activerecord-copy.gemspec +++ b/activerecord-copy.gemspec @@ -19,8 +19,14 @@ Gem::Specification.new do |gem| gem.require_paths = ['lib'] gem.add_dependency('activerecord', '>= 3.1') + gem.add_dependency('pg', '>= 1.3.0') gem.add_development_dependency('rspec', '>= 2.12.0') gem.add_development_dependency('rspec-core', '>= 2.12.0') + gem.add_development_dependency('rspec-rails') + + gem.add_development_dependency('rails', '>= 6.1.4.4') + gem.add_development_dependency('activerecord-postgis-adapter', '~> 7.1') + gem.add_development_dependency('dotenv-rails', '>= 2.7.6') gem.add_development_dependency('rgeo', '>= 2.4.0') -end +end \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..42d3892 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3" +services: + db: + image: postgres/postgres + ports: + - "5577:5432" + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_HOST_AUTH_METHOD: "trust" + diff --git a/lib/activerecord-copy.rb b/lib/activerecord-copy.rb index e714574..f3d817d 100644 --- a/lib/activerecord-copy.rb +++ b/lib/activerecord-copy.rb @@ -1,124 +1,31 @@ -require 'activerecord-copy/version' +require 'json' +require 'active_record' +require 'activerecord-copy/version' require 'activerecord-copy/constants' require 'activerecord-copy/exception' require 'activerecord-copy/temp_buffer' - require 'activerecord-copy/encode_for_copy' -require 'activerecord-copy/decoder' - -require 'json' - -require 'active_record' +require 'activerecord-copy/column_helper' module ActiveRecordCopy module CopyFromClient extend ActiveSupport::Concern - class CopyHandler - def initialize(columns:, model_class:, table_name:, send_at_once:) - @columns = columns - @model_class = model_class - @connection = model_class.connection.raw_connection - @table_name = table_name - @send_at_once = send_at_once - @column_types = columns.map do |c| - column = model_class.columns_hash[c.to_s] - raise format('Could not find column %s on %s', c, model_class.table_name) if column.nil? - - if column.type == :integer - if column.limit == 8 - :bigint - elsif column.limit == 2 - :smallint - else - :integer - end - elsif column.sql_type == 'real' - :real - else - column.type - end - end - end - - def <<(row) - start_copy_if_needed - @encoder.add row - unless @send_at_once - @connection.put_copy_data(@encoder.get_intermediate_io) - end - end - - def copy(&_block) - reset - - if @send_at_once - yield(self) - run_copy_at_once - return - end - - begin - yield(self) - rescue Exception => err - if @copy_initialized - errmsg = format('%s while copy data: %s', err.class.name, err.message) - @connection.put_copy_end(errmsg) - @connection.get_result - end - raise - else - if @copy_initialized - @encoder.remove # writes the end marker - @connection.put_copy_end - @connection.get_last_result - end - end - end - - private - - def start_copy_if_needed - return if @copy_initialized || @send_at_once - - @connection.exec(copy_sql) - @copy_initialized = true - end - - def copy_sql - %{COPY #{@table_name}("#{@columns.join('","')}") FROM STDIN BINARY} - end - - def run_copy_at_once - io = @encoder.get_io - - @connection.copy_data(copy_sql) do - begin - while chunk = io.readpartial(10_240) # rubocop:disable Lint/AssignmentInCondition - @connection.put_copy_data chunk - end - rescue EOFError # rubocop:disable Lint/HandleExceptions - end - end - - @encoder.remove - - nil - end - - def reset - @encoder = ActiveRecordCopy::EncodeForCopy.new column_types: @column_types - @copy_initialized = false - end - end - class_methods do - def copy_from_client(columns, table_name: nil, send_at_once: false, &block) + def copy_from_client(columns, table_name: nil, &block) table_name ||= quoted_table_name - handler = CopyHandler.new(columns: columns, model_class: self, table_name: table_name, send_at_once: send_at_once) - handler.copy(&block) - true + connection = self.connection.raw_connection + column_types = columns.map do |c| + column = self.columns_hash[c.to_s] + raise format('Could not find column %s on %s', c, self.table_name) if column.nil? + ColumnHelper.find_column_type(column) + end + sql = %{COPY #{table_name}("#{columns.join('","')}") FROM STDIN BINARY} + encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: column_types, connection: connection) + connection.copy_data(sql) do + encoder.process(&block) + end end end end diff --git a/lib/activerecord-copy/column_helper.rb b/lib/activerecord-copy/column_helper.rb new file mode 100644 index 0000000..8588992 --- /dev/null +++ b/lib/activerecord-copy/column_helper.rb @@ -0,0 +1,19 @@ +module ActiveRecordCopy + class ColumnHelper + def self.find_column_type(column) + if column.type == :integer + if column.limit == 8 + :bigint + elsif column.limit == 2 + :smallint + else + :integer + end + elsif column.sql_type == 'real' + :real + else + column.type + end + end + end +end \ No newline at end of file diff --git a/lib/activerecord-copy/decoder.rb b/lib/activerecord-copy/decoder.rb deleted file mode 100644 index da70654..0000000 --- a/lib/activerecord-copy/decoder.rb +++ /dev/null @@ -1,176 +0,0 @@ -require 'tempfile' -require 'stringio' - -module ActiveRecordCopy - class Decoder - def initialize(options = {}) - @options = options - @closed = false - if options[:column_types].is_a?(Array) - map = {} - options[:column_types].each_with_index do |c, i| - map[i] = c - end - options[:column_types] = map - else - options[:column_types] ||= {} - end - @io = nil - end - - def read_line - return nil if @closed - setup_io unless @io - row = [] - bytes = @io.read(2) - # p bytes - column_count = bytes.unpack(PACKED_UINT_16).first - if column_count == 65_535 - @closed = true - return nil - end - # @io.write([row.size].pack(PACKED_UINT_32)) - 0.upto(column_count - 1).each do |index| - field = decode_field(@io) - row[index] = if field.nil? - field - elsif @options[:column_types][index] - map_field(field, @options[:column_types][index]) - else - field - end - end - row - end - - def each - loop do - result = read_line - break unless result - yield result - break if @closed - end - end - - private - - def setup_io - if @options[:file] - @io = File.open(@options[:file], 'r:' + ASCII_8BIT_ENCODING) - elsif !@options[:io].nil? - @io = @options[:io] - else - raise 'NO io present' - end - header = "PGCOPY\n\377\r\n\0".force_encoding(ASCII_8BIT_ENCODING) - result = @io.read(header.bytesize) - raise 'invalid format' if result != header - # p @io.read(10) - - @io.read(2) # blank - @io.read(6) # blank - end - - def decode_field(io) - bytes = io.read(4) - - if bytes == "\xFF\xFF\xFF\xFF".force_encoding(ASCII_8BIT_ENCODING) - return nil - else - io.read(bytes.unpack(PACKED_UINT_32).first) - end - end - - def map_field(data, type) - # p [type, data] - - case type - when :int, :integer - data.unpack(PACKED_UINT_32).first - when :bytea - data - when :bigint - data.unpack(PACKED_UINT_64).first - when :float, :double - data.unpack(PACKED_FLOAT_64).first - when :boolean - v = data.unpack(PACKED_UINT_8).first - v == 1 - when :string, :text, :character - data.force_encoding(UTF_8_ENCODING) - when :json - JSON.load(data) - when :uuid - r = data.unpack('H*').first - "#{r[0..7]}-#{r[8..11]}-#{r[12..15]}-#{r[16..19]}-#{r[20..-1]}" - when :uuid_raw - r = data.unpack('H*').first - when :array, :"integer[]", :"uuid[]", :"character[]" - io = StringIO.new(data) - io.read(4) # unknown - io.read(4) # unknown - atype_raw = io.read(4) - return [] if atype_raw.nil? - atype = atype_raw.unpack(PACKED_UINT_32).first # string type? - return [] if io.pos == io.size - size = io.read(4).unpack(PACKED_UINT_32).first - io.read(4) # should be 1 for dimension - # p [atype, size] - # p data - case atype - when UUID_TYPE_OID - 0.upto(size - 1).map do - io.read(4) # size - r = io.read(16).unpack(PACKED_HEX_STRING).first - "#{r[0..7]}-#{r[8..11]}-#{r[12..15]}-#{r[16..19]}-#{r[20..-1]}" - end - when TEXT_TYPE_OID, VARCHAR_TYPE_OID - 0.upto(size - 1).map do - size = io.read(4).unpack(PACKED_UINT_32).first - io.read(size) - end - when INT_TYPE_OID - 0.upto(size - 1).map do - size = io.read(4).unpack(PACKED_UINT_32).first - bytes = io.read(size) - bytes.unpack(PACKED_UINT_32).first - end - else - raise "Unsupported Array type #{atype}" - end - when :hstore, :hash - io = StringIO.new(data) - fields = io.read(4).unpack(PACKED_UINT_32).first - h = {} - - 0.upto(fields - 1).each do - key_size = io.read(4).unpack(PACKED_UINT_32).first - key = io.read(key_size).force_encoding("UTF-8") - value_size = io.read(4).unpack(PACKED_UINT_32).first - if value_size == 4294967295 # nil "\xFF\xFF\xFF\xFF" - value = nil - else - value = io.read(value_size) - value = value.force_encoding("UTF-8") if !value.nil? - end - h[key] = value - end - raise "remaining hstore bytes!" if io.pos != io.size - h - when :time, :timestamp - d = data.unpack("L!>").first - Time.at((d + POSTGRES_EPOCH_TIME) / 1_000_000.0).utc - when :date - # couldn't find another way to get signed network byte order - m = 0b0111_1111_1111_1111_1111_1111_1111_1111 - d = data.unpack(PACKED_UINT_32).first - d = (d & m) - m - 1 if data.bytes[0] & 0b1000_0000 > 0 # negative number - - # p [data, d, Date.jd(d + Date.new(2000,1,1).jd)] - Date.jd(d + Date.new(2000, 1, 1).jd) - else - raise "Unsupported format #{type}" - end - end - end -end diff --git a/lib/activerecord-copy/encode_for_copy.rb b/lib/activerecord-copy/encode_for_copy.rb index 4afc2fb..0a458c8 100644 --- a/lib/activerecord-copy/encode_for_copy.rb +++ b/lib/activerecord-copy/encode_for_copy.rb @@ -3,91 +3,53 @@ require 'tempfile' require 'stringio' require 'ipaddr' +require 'active_record' +require_relative 'mac_address' module ActiveRecordCopy - class EncodeForCopy - def initialize(options = {}) - @options = options - @closed = false - @column_types = @options[:column_types] || {} - @io = nil - @buffer = TempBuffer.new - @row_size_encoded = nil - end - - def add(row) - fail ArgumentError.new('Empty row added') if row.empty? - setup_io unless @io - - # Row size needs to be the same across all rows, so its safe to cache this - @row_size_encoded ||= [row.size].pack(PACKED_UINT_16) - write(@io, @row_size_encoded) - - row.each_with_index do |col, index| - encode_field(@buffer, col, index) - end - - write(@io, @buffer.read) - - @buffer.reopen + class ConnectionWriter + def initialize(connection) + @connection = connection end - ROW_END_MARKER = [-1].pack(PACKED_UINT_16) - def close - raise(Exception, 'No rows have been added to the encoder!') if @io.nil? - - @closed = true - unless @buffer.empty? - write(@io, @buffer.read) - @buffer.reopen - end - write(@io, ROW_END_MARKER) - @io.rewind + def flush + @connection.flush end - def get_io - close unless @closed - @io + def write(buf) + @connection.sync_put_copy_data(buf) end + end - def get_intermediate_io - unless @buffer.empty? - write(@io, @buffer.read) - @buffer.reopen - end - @io.rewind - buf = @io.read - if @options[:use_tempfile] == true - remove - @io = Tempfile.new('copy_binary', encoding: ASCII_8BIT_ENCODING) - @io.unlink unless @options[:skip_unlink] == true - else - @io = StringIO.new - end - buf + class EncodeForCopy + def initialize(column_types:, connection:) + @column_types = column_types + @row_size_encoded = [column_types.size].pack(PACKED_UINT_16) + @io = ConnectionWriter.new(connection) end - def remove - return unless @io.is_a?(Tempfile) - - @io.close - @io.unlink + def process(&_block) + write(@io, "PGCOPY\n\377\r\n\0") + pack_and_write(@io, [0, 0], PACKED_UINT_32 + PACKED_UINT_32) + yield self + @io.flush end - private + def <<(row) + self.add(row) + end - def setup_io - if @options[:use_tempfile] == true - @io = Tempfile.new('copy_binary', encoding: ASCII_8BIT_ENCODING) - @io.unlink unless @options[:skip_unlink] == true - else - @io = StringIO.new + def add(row) + fail ArgumentError.new('Empty row added') if row.empty? + write(@io, @row_size_encoded) + row.each_with_index do |col, index| + encode_field(@io, col, index) end - write(@io, "PGCOPY\n\377\r\n\0") - pack_and_write(@io, [0, 0], PACKED_UINT_32 + PACKED_UINT_32) end + private + def pack_and_write(io, data, pack_format) if io.is_a?(String) data.pack(pack_format, buffer: io) @@ -110,6 +72,7 @@ def write(io, buf) BUFSIZE_4 = [4].pack(PACKED_UINT_32) BUFSIZE_8 = [8].pack(PACKED_UINT_32) BUFSIZE_16 = [16].pack(PACKED_UINT_32) + def write_with_bufsize(io, buf) case buf.bytesize when 4 @@ -133,17 +96,20 @@ def write_simple_field(io, field, type) pack_and_write_with_bufsize(io, [field.to_i], PACKED_UINT_32) when :smallint pack_and_write_with_bufsize(io, [field.to_i], PACKED_UINT_16) - when :numeric + when :numeric, :decimal encode_numeric(io, field) when :real pack_and_write_with_bufsize(io, [field], PACKED_FLOAT_32) when :float pack_and_write_with_bufsize(io, [field], PACKED_FLOAT_64) + when :time + data = (field - field.beginning_of_day) + pack_and_write_with_bufsize(io, [data.to_i * 1000000], PACKED_UINT_64) when :timestamp, :timestamptz data = field.tv_sec * 1_000_000 + field.tv_usec - POSTGRES_EPOCH_TIME pack_and_write_with_bufsize(io, [data.to_i], PACKED_UINT_64) when :date - data = field - Date.new(2000, 1, 1) + data = field.to_date - Date.new(2000, 1, 1) pack_and_write_with_bufsize(io, [data.to_i], PACKED_UINT_32) else raise Exception, "Unsupported simple type: #{type}" @@ -151,6 +117,7 @@ def write_simple_field(io, field, type) end NIL_FIELD = [-1].pack(PACKED_UINT_32) + def encode_field(io, field, index, depth = 0) # Nil is an exception in that any kind of field type can have a nil value transmitted if field.nil? @@ -164,12 +131,17 @@ def encode_field(io, field, index, depth = 0) end case @column_types[index] - when :bigint, :integer, :smallint, :numeric, :float, :real + when :bigint, :integer, :smallint, :numeric, :float, :real, :decimal, :date, :time write_simple_field(io, field, @column_types[index]) when :uuid pack_and_write_with_bufsize(io, [field.delete('-')], PACKED_HEX_STRING) - when :inet - encode_ip_addr(io, IPAddr.new(field)) + when :inet, :cidr + if String === field + field = IPAddr.new(field) + end + encode_ip_addr(io, field) + when :macaddr + write_with_bufsize(io, MacAddress.new(field).to_bytes) when :binary write_with_bufsize(io, field.dup) when :json @@ -372,7 +344,7 @@ def base10_to_base10000(intval) NUMERIC_DEC_DIGITS = 4 def encode_numeric(io, field) - float_str = field.to_s + float_str = field.abs.to_s digits_base10 = float_str.scan(/\d/).map(&:to_i) weight_base10 = float_str.index('.') sign = field < 0.0 ? 0x4000 : 0 diff --git a/lib/activerecord-copy/mac_address.rb b/lib/activerecord-copy/mac_address.rb new file mode 100644 index 0000000..08b7255 --- /dev/null +++ b/lib/activerecord-copy/mac_address.rb @@ -0,0 +1,23 @@ +module ActiveRecordCopy + class MacAddress + def initialize(str) + str = str.strip + if !self.class.validate_strict(str) + raise ArgumentError.new("Invalid MAC address: #{str}") + end + @bytes = str.split(/[-,:]/).map { |s| s.to_i(16) } + end + + def to_s + @bytes.map { |h| h.hex }.join(":") + end + + def to_bytes + @bytes.pack('C*') + end + + def self.validate_strict(mac) + !!(mac =~ /^([0-9a-fA-F]{2}[:-]){5}[0-9a-fA-F]{2}$/i) + end + end +end \ No newline at end of file diff --git a/log/development.log b/log/development.log new file mode 100644 index 0000000..e69de29 diff --git a/spec/big_write_spec.rb b/spec/big_write_spec.rb deleted file mode 100644 index efeb71e..0000000 --- a/spec/big_write_spec.rb +++ /dev/null @@ -1,17 +0,0 @@ -require File.expand_path(File.dirname(__FILE__) + '/spec_helper') -require 'benchmark' - -describe 'testing changes with large imports and speed issues' do - it 'imports lots of data quickly' do - encoder = ActiveRecordCopy::EncodeForCopy.new(temp_file: true) - - puts Benchmark.measure { - 0.upto(100_000) do - encoder.add [1, 'text', { a: 1, b: 'asdf' }] - end - } - - encoder.close - _ = encoder.get_io - end -end diff --git a/spec/dummy/.ruby-version b/spec/dummy/.ruby-version new file mode 100644 index 0000000..a603bb5 --- /dev/null +++ b/spec/dummy/.ruby-version @@ -0,0 +1 @@ +2.7.5 diff --git a/spec/dummy/Gemfile b/spec/dummy/Gemfile new file mode 100644 index 0000000..002f058 --- /dev/null +++ b/spec/dummy/Gemfile @@ -0,0 +1,10 @@ +source 'https://rubygems.org' + +ruby '2.7.5' + +gem 'rails', '~> 6.1.4', '>= 6.1.4.4' + +# Windows does not include zoneinfo files, so bundle the tzinfo-data gem +gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby] +gem "pg", "~> 1.3" +gem "activerecord-postgis-adapter", "~> 7.1" diff --git a/spec/dummy/README.md b/spec/dummy/README.md new file mode 100644 index 0000000..63435ec --- /dev/null +++ b/spec/dummy/README.md @@ -0,0 +1,10 @@ +# Dummy Rails App + +Dummy rails app use for the integration tests. +It requires postgres to be running (see [start-local.sh](../../start-local-db.sh)) + +Set up the DB: +``` +bundle install +DATABASE_URL='postgis://postgres:postgres@localhost:5577' rake db:create db:migrate db:test:prepare +``` \ No newline at end of file diff --git a/spec/dummy/Rakefile b/spec/dummy/Rakefile new file mode 100644 index 0000000..9a5ea73 --- /dev/null +++ b/spec/dummy/Rakefile @@ -0,0 +1,6 @@ +# Add your own tasks in files placed in lib/tasks ending in .rake, +# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake. + +require_relative "config/application" + +Rails.application.load_tasks diff --git a/spec/dummy/app/models/my_model.rb b/spec/dummy/app/models/my_model.rb new file mode 100644 index 0000000..116b5f9 --- /dev/null +++ b/spec/dummy/app/models/my_model.rb @@ -0,0 +1,2 @@ +class MyModel < ActiveRecord::Base +end diff --git a/spec/dummy/bin/bundle b/spec/dummy/bin/bundle new file mode 100644 index 0000000..5b593cb --- /dev/null +++ b/spec/dummy/bin/bundle @@ -0,0 +1,114 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# +# This file was generated by Bundler. +# +# The application 'bundle' is installed as part of a gem, and +# this file is here to facilitate running it. +# + +require "rubygems" + +m = Module.new do + module_function + + def invoked_as_script? + File.expand_path($0) == File.expand_path(__FILE__) + end + + def env_var_version + ENV["BUNDLER_VERSION"] + end + + def cli_arg_version + return unless invoked_as_script? # don't want to hijack other binstubs + return unless "update".start_with?(ARGV.first || " ") # must be running `bundle update` + bundler_version = nil + update_index = nil + ARGV.each_with_index do |a, i| + if update_index && update_index.succ == i && a =~ Gem::Version::ANCHORED_VERSION_PATTERN + bundler_version = a + end + next unless a =~ /\A--bundler(?:[= ](#{Gem::Version::VERSION_PATTERN}))?\z/ + bundler_version = $1 + update_index = i + end + bundler_version + end + + def gemfile + gemfile = ENV["BUNDLE_GEMFILE"] + return gemfile if gemfile && !gemfile.empty? + + File.expand_path("../../Gemfile", __FILE__) + end + + def lockfile + lockfile = + case File.basename(gemfile) + when "gems.rb" then gemfile.sub(/\.rb$/, gemfile) + else "#{gemfile}.lock" + end + File.expand_path(lockfile) + end + + def lockfile_version + return unless File.file?(lockfile) + lockfile_contents = File.read(lockfile) + return unless lockfile_contents =~ /\n\nBUNDLED WITH\n\s{2,}(#{Gem::Version::VERSION_PATTERN})\n/ + Regexp.last_match(1) + end + + def bundler_requirement + @bundler_requirement ||= + env_var_version || cli_arg_version || + bundler_requirement_for(lockfile_version) + end + + def bundler_requirement_for(version) + return "#{Gem::Requirement.default}.a" unless version + + bundler_gem_version = Gem::Version.new(version) + + requirement = bundler_gem_version.approximate_recommendation + + return requirement unless Gem.rubygems_version < Gem::Version.new("2.7.0") + + requirement += ".a" if bundler_gem_version.prerelease? + + requirement + end + + def load_bundler! + ENV["BUNDLE_GEMFILE"] ||= gemfile + + activate_bundler + end + + def activate_bundler + gem_error = activation_error_handling do + gem "bundler", bundler_requirement + end + return if gem_error.nil? + require_error = activation_error_handling do + require "bundler/version" + end + return if require_error.nil? && Gem::Requirement.new(bundler_requirement).satisfied_by?(Gem::Version.new(Bundler::VERSION)) + warn "Activating bundler (#{bundler_requirement}) failed:\n#{gem_error.message}\n\nTo install the version of bundler this project requires, run `gem install bundler -v '#{bundler_requirement}'`" + exit 42 + end + + def activation_error_handling + yield + nil + rescue StandardError, LoadError => e + e + end +end + +m.load_bundler! + +if m.invoked_as_script? + load Gem.bin_path("bundler", "bundle") +end diff --git a/spec/dummy/bin/rails b/spec/dummy/bin/rails new file mode 100755 index 0000000..efc0377 --- /dev/null +++ b/spec/dummy/bin/rails @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +APP_PATH = File.expand_path("../config/application", __dir__) +require_relative "../config/boot" +require "rails/commands" diff --git a/spec/dummy/bin/rake b/spec/dummy/bin/rake new file mode 100644 index 0000000..e69de29 diff --git a/spec/dummy/bin/setup b/spec/dummy/bin/setup new file mode 100644 index 0000000..e69de29 diff --git a/spec/dummy/bin/spring b/spec/dummy/bin/spring new file mode 100644 index 0000000..e69de29 diff --git a/spec/dummy/config.ru b/spec/dummy/config.ru new file mode 100644 index 0000000..4a3c09a --- /dev/null +++ b/spec/dummy/config.ru @@ -0,0 +1,6 @@ +# This file is used by Rack-based servers to start the application. + +require_relative "config/environment" + +run Rails.application +Rails.application.load_server diff --git a/spec/dummy/config/application.rb b/spec/dummy/config/application.rb new file mode 100644 index 0000000..216b5ea --- /dev/null +++ b/spec/dummy/config/application.rb @@ -0,0 +1,40 @@ +require_relative "boot" + +require "rails" +# Pick the frameworks you want: +require "active_model/railtie" +require "active_job/railtie" +require "active_record/railtie" +require "active_storage/engine" +require "action_controller/railtie" +require "action_mailer/railtie" +require "action_mailbox/engine" +require "action_text/engine" +require "action_view/railtie" +require "action_cable/engine" +# require "sprockets/railtie" +require "rails/test_unit/railtie" + +# Require the gems listed in Gemfile, including any gems +# you've limited to :test, :development, or :production. +Bundler.require(*Rails.groups) + +module Dummy + class Application < Rails::Application + # Initialize configuration defaults for originally generated Rails version. + config.load_defaults 6.1 + + # Configuration for the application, engines, and railties goes here. + # + # These settings can be overridden in specific environments using the files + # in config/environments, which are processed later. + # + # config.time_zone = "Central Time (US & Canada)" + # config.eager_load_paths << Rails.root.join("extras") + + # Only loads a smaller set of middleware suitable for API only apps. + # Middleware like session, flash, cookies can be added back manually. + # Skip views, helpers and assets when generating a new resource. + config.api_only = true + end +end diff --git a/spec/dummy/config/boot.rb b/spec/dummy/config/boot.rb new file mode 100644 index 0000000..d69bd27 --- /dev/null +++ b/spec/dummy/config/boot.rb @@ -0,0 +1,3 @@ +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__) + +require "bundler/setup" # Set up gems listed in the Gemfile. diff --git a/spec/dummy/config/database.yml b/spec/dummy/config/database.yml new file mode 100644 index 0000000..a191d48 --- /dev/null +++ b/spec/dummy/config/database.yml @@ -0,0 +1,67 @@ +default: &default + adapter: postgresql + encoding: unicode + # For details on connection pooling, see Rails configuration guide + # https://guides.rubyonrails.org/configuring.html#database-pooling + pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + +development: + <<: *default + url: <%= ENV['DATABASE_URL'] %> + database: dummy_development + + # The specified database role being used to connect to postgres. + # To create additional roles in postgres see `$ createuser --help`. + # When left blank, postgres will use the default role. This is + # the same name as the operating system user that initialized the database. + #username: stimulus_reflex_expo + # The password associated with the postgres role (username). + #password: + # Connect on a TCP socket. Omitted by default since the client uses a + # domain socket that doesn't need configuration. Windows does not have + # domain sockets, so uncomment these lines. + #host: localhost + # The TCP port the server listens on. Defaults to 5432. + # If your server runs on a different port number, change accordingly. + #port: 5432 + # Schema search path. The server defaults to $user,public + #schema_search_path: myapp,sharedapp,public + # Minimum log levels, in increasing order: + # debug5, debug4, debug3, debug2, debug1, + # log, notice, warning, error, fatal, and panic + # Defaults to warning. + #min_messages: notice + +# Warning: The database defined as "test" will be erased and +# re-generated from your development database when you run "rake". +# Do not set this db to the same as development or production. +test: + <<: *default + url: <%= ENV['DATABASE_URL'] %> + database: dummy_test + +# As with config/credentials.yml, you never want to store sensitive information, +# like your database password, in your source code. If your source code is +# ever seen by anyone, they now have access to your database. +# +# Instead, provide the password as a unix environment variable when you boot +# the app. Read https://guides.rubyonrails.org/configuring.html#configuring-a-database +# for a full rundown on how to provide these environment variables in a +# production deployment. +# +# On Heroku and other platform providers, you may have a full connection URL +# available as an environment variable. For example: +# +# DATABASE_URL="postgres://myuser:mypass@localhost/somedatabase" +# +# You can use this database configuration with: +# +# production: +# url: <%= ENV['DATABASE_URL'] %> +# + +# https://devcenter.heroku.com/articles/rails-database-connection-behavior +production: + <<: *default + url: <%= ENV['DATABASE_URL']&.sub(/^postgres/, "postgis") %> + database: dummy_production diff --git a/spec/dummy/config/environment.rb b/spec/dummy/config/environment.rb new file mode 100644 index 0000000..cac5315 --- /dev/null +++ b/spec/dummy/config/environment.rb @@ -0,0 +1,5 @@ +# Load the Rails application. +require_relative "application" + +# Initialize the Rails application. +Rails.application.initialize! diff --git a/spec/dummy/config/environments/development.rb b/spec/dummy/config/environments/development.rb new file mode 100644 index 0000000..eb2744f --- /dev/null +++ b/spec/dummy/config/environments/development.rb @@ -0,0 +1,65 @@ +require "active_support/core_ext/integer/time" + +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # In the development environment your application's code is reloaded any time + # it changes. This slows down response time but is perfect for development + # since you don't have to restart the web server when you make code changes. + config.cache_classes = false + + # Do not eager load code on boot. + config.eager_load = false + + # Show full error reports. + config.consider_all_requests_local = true + + # Enable/disable caching. By default caching is disabled. + # Run rails dev:cache to toggle caching. + if Rails.root.join('tmp', 'caching-dev.txt').exist? + config.cache_store = :memory_store + config.public_file_server.headers = { + 'Cache-Control' => "public, max-age=#{2.days.to_i}" + } + else + config.action_controller.perform_caching = false + + config.cache_store = :null_store + end + + # Store uploaded files on the local file system (see config/storage.yml for options). + config.active_storage.service = :local + + # Don't care if the mailer can't send. + config.action_mailer.raise_delivery_errors = false + + config.action_mailer.perform_caching = false + + # Print deprecation notices to the Rails logger. + config.active_support.deprecation = :log + + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Raise an error on page load if there are pending migrations. + config.active_record.migration_error = :page_load + + # Highlight code that triggered database queries in logs. + config.active_record.verbose_query_logs = true + + + # Raises error for missing translations. + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true + + # Use an evented file watcher to asynchronously detect changes in source code, + # routes, locales, etc. This feature depends on the listen gem. + + # Uncomment if you wish to allow Action Cable access from any origin. + # config.action_cable.disable_request_forgery_protection = true +end diff --git a/spec/dummy/config/environments/production.rb b/spec/dummy/config/environments/production.rb new file mode 100644 index 0000000..b564331 --- /dev/null +++ b/spec/dummy/config/environments/production.rb @@ -0,0 +1,113 @@ +require "active_support/core_ext/integer/time" + +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # Code is not reloaded between requests. + config.cache_classes = true + + # Eager load code on boot. This eager loads most of Rails and + # your application in memory, allowing both threaded web servers + # and those relying on copy on write to perform better. + # Rake tasks automatically ignore this option for performance. + config.eager_load = true + + # Full error reports are disabled and caching is turned on. + config.consider_all_requests_local = false + + # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] + # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). + # config.require_master_key = true + + # Disable serving static files from the `/public` folder by default since + # Apache or NGINX already handles this. + config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? + + # Enable serving of images, stylesheets, and JavaScripts from an asset server. + # config.asset_host = 'http://assets.example.com' + + # Specifies the header that your server uses for sending files. + # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache + # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + + # Store uploaded files on the local file system (see config/storage.yml for options). + config.active_storage.service = :local + + # Mount Action Cable outside main process or domain. + # config.action_cable.mount_path = nil + # config.action_cable.url = 'wss://example.com/cable' + # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] + + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. + # config.force_ssl = true + + # Include generic and useful information about system operation, but avoid logging too much + # information to avoid inadvertent exposure of personally identifiable information (PII). + config.log_level = :info + + # Prepend all log lines with the following tags. + config.log_tags = [ :request_id ] + + # Use a different cache store in production. + # config.cache_store = :mem_cache_store + + # Use a real queuing backend for Active Job (and separate queues per environment). + # config.active_job.queue_adapter = :resque + # config.active_job.queue_name_prefix = "dummy_production" + + config.action_mailer.perform_caching = false + + # Ignore bad email addresses and do not raise email delivery errors. + # Set this to true and configure the email server for immediate delivery to raise delivery errors. + # config.action_mailer.raise_delivery_errors = false + + # Enable locale fallbacks for I18n (makes lookups for any locale fall back to + # the I18n.default_locale when a translation cannot be found). + config.i18n.fallbacks = true + + # Send deprecation notices to registered listeners. + config.active_support.deprecation = :notify + + # Log disallowed deprecations. + config.active_support.disallowed_deprecation = :log + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Use default logging formatter so that PID and timestamp are not suppressed. + config.log_formatter = ::Logger::Formatter.new + + # Use a different logger for distributed setups. + # require "syslog/logger" + # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') + + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end + + # Do not dump schema after migrations. + config.active_record.dump_schema_after_migration = false + + # Inserts middleware to perform automatic connection switching. + # The `database_selector` hash is used to pass options to the DatabaseSelector + # middleware. The `delay` is used to determine how long to wait after a write + # to send a subsequent read to the primary. + # + # The `database_resolver` class is used by the middleware to determine which + # database is appropriate to use based on the time delay. + # + # The `database_resolver_context` class is used by the middleware to set + # timestamps for the last write to the primary. The resolver uses the context + # class timestamps to determine how long to wait before reading from the + # replica. + # + # By default Rails will store a last write timestamp in the session. The + # DatabaseSelector middleware is designed as such you can define your own + # strategy for connection switching and pass that into the middleware through + # these configuration options. + # config.active_record.database_selector = { delay: 2.seconds } + # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver + # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session +end diff --git a/spec/dummy/config/environments/test.rb b/spec/dummy/config/environments/test.rb new file mode 100644 index 0000000..93ed4f1 --- /dev/null +++ b/spec/dummy/config/environments/test.rb @@ -0,0 +1,60 @@ +require "active_support/core_ext/integer/time" + +# The test environment is used exclusively to run your application's +# test suite. You never need to work with it otherwise. Remember that +# your test database is "scratch space" for the test suite and is wiped +# and recreated between test runs. Don't rely on the data there! + +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + config.cache_classes = false + config.action_view.cache_template_loading = true + + # Do not eager load code on boot. This avoids loading your whole application + # just for the purpose of running a single test. If you are using a tool that + # preloads Rails for running tests, you may have to set it to true. + config.eager_load = false + + # Configure public file server for tests with Cache-Control for performance. + config.public_file_server.enabled = true + config.public_file_server.headers = { + 'Cache-Control' => "public, max-age=#{1.hour.to_i}" + } + + # Show full error reports and disable caching. + config.consider_all_requests_local = true + config.action_controller.perform_caching = false + config.cache_store = :null_store + + # Raise exceptions instead of rendering exception templates. + config.action_dispatch.show_exceptions = false + + # Disable request forgery protection in test environment. + config.action_controller.allow_forgery_protection = false + + # Store uploaded files on the local file system in a temporary directory. + config.active_storage.service = :test + + config.action_mailer.perform_caching = false + + # Tell Action Mailer not to deliver emails to the real world. + # The :test delivery method accumulates sent emails in the + # ActionMailer::Base.deliveries array. + config.action_mailer.delivery_method = :test + + # Print deprecation notices to the stderr. + config.active_support.deprecation = :stderr + + # Raise exceptions for disallowed deprecations. + config.active_support.disallowed_deprecation = :raise + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Raises error for missing translations. + # config.i18n.raise_on_missing_translations = true + + # Annotate rendered view with file names. + # config.action_view.annotate_rendered_view_with_filenames = true +end diff --git a/spec/dummy/db/migrate/20220316104731_create_my_models.rb b/spec/dummy/db/migrate/20220316104731_create_my_models.rb new file mode 100644 index 0000000..1f3ed62 --- /dev/null +++ b/spec/dummy/db/migrate/20220316104731_create_my_models.rb @@ -0,0 +1,32 @@ +class CreateMyModels < ActiveRecord::Migration[6.1] + def change + create_table :my_models do |t| + t.binary :binary + t.boolean :boolean + t.date :date + t.datetime :datetime + t.decimal :decimal + t.float :float + t.integer :integer + t.bigint :bigint + t.string :string + t.text :text + t.time :time + t.timestamp :timestamp + t.geometry :geometry + + t.json :json + t.jsonb :jsonb + + t.inet :inet + t.cidr :cidr + t.macaddr :macaddr + + t.int4range :int4range + t.numrange :numrange + t.tstzrange :tstzrange + t.daterange :daterange + end + end + +end diff --git a/spec/dummy/db/schema.rb b/spec/dummy/db/schema.rb new file mode 100644 index 0000000..e98e677 --- /dev/null +++ b/spec/dummy/db/schema.rb @@ -0,0 +1,44 @@ +# This file is auto-generated from the current state of the database. Instead +# of editing this file, please use the migrations feature of Active Record to +# incrementally modify your database, and then regenerate this schema definition. +# +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. +# +# It's strongly recommended that you check this file into your version control system. + +ActiveRecord::Schema.define(version: 2022_03_16_104731) do + + # These are extensions that must be enabled in order to support this database + enable_extension "plpgsql" + enable_extension "postgis" + + create_table "my_models", force: :cascade do |t| + t.binary "binary" + t.boolean "boolean" + t.date "date" + t.datetime "datetime" + t.decimal "decimal" + t.float "float" + t.integer "integer" + t.bigint "bigint" + t.string "string" + t.text "text" + t.time "time" + t.datetime "timestamp" + t.geometry "geometry", limit: {:srid=>0, :type=>"geometry"} + t.json "json" + t.jsonb "jsonb" + t.inet "inet" + t.cidr "cidr" + t.macaddr "macaddr" + t.int4range "int4range" + t.numrange "numrange" + t.tstzrange "tstzrange" + t.daterange "daterange" + end + +end diff --git a/spec/errors_spec.rb b/spec/errors_spec.rb deleted file mode 100644 index 3540694..0000000 --- a/spec/errors_spec.rb +++ /dev/null @@ -1,8 +0,0 @@ -require File.expand_path(File.dirname(__FILE__) + '/spec_helper') - -describe 'throwing errors' do - it 'raises an error when no rows have been added to the encoder' do - encoder = ActiveRecordCopy::EncodeForCopy.new - expect { encoder.close }.to raise_error(ActiveRecordCopy::Exception) - end -end diff --git a/spec/fixtures/3_col_array.txt b/spec/fixtures/3_col_array.txt deleted file mode 100644 index 13a605c..0000000 --- a/spec/fixtures/3_col_array.txt +++ /dev/null @@ -1 +0,0 @@ -1 hi {hi,there,rubyist} diff --git a/spec/fixtures/3_col_hstore.dat b/spec/fixtures/3_col_hstore.dat deleted file mode 100644 index c0bdc4e..0000000 Binary files a/spec/fixtures/3_col_hstore.dat and /dev/null differ diff --git a/spec/fixtures/3_col_hstore.txt b/spec/fixtures/3_col_hstore.txt deleted file mode 100644 index 6fd729c..0000000 --- a/spec/fixtures/3_col_hstore.txt +++ /dev/null @@ -1 +0,0 @@ -1 text "a"=>"1", "b"=>"asdf" diff --git a/spec/fixtures/3_column_array.dat b/spec/fixtures/3_column_array.dat deleted file mode 100644 index 83ae5c4..0000000 Binary files a/spec/fixtures/3_column_array.dat and /dev/null differ diff --git a/spec/fixtures/array_with_two.dat b/spec/fixtures/array_with_two.dat deleted file mode 100644 index b7a18df..0000000 Binary files a/spec/fixtures/array_with_two.dat and /dev/null differ diff --git a/spec/fixtures/array_with_two2.dat b/spec/fixtures/array_with_two2.dat deleted file mode 100644 index 74d9fbb..0000000 Binary files a/spec/fixtures/array_with_two2.dat and /dev/null differ diff --git a/spec/fixtures/big_str_array.dat b/spec/fixtures/big_str_array.dat deleted file mode 100644 index eec0413..0000000 Binary files a/spec/fixtures/big_str_array.dat and /dev/null differ diff --git a/spec/fixtures/big_str_array2.dat b/spec/fixtures/big_str_array2.dat deleted file mode 100644 index 76d1ba8..0000000 Binary files a/spec/fixtures/big_str_array2.dat and /dev/null differ diff --git a/spec/fixtures/bigint.dat b/spec/fixtures/bigint.dat deleted file mode 100644 index 094cc45..0000000 Binary files a/spec/fixtures/bigint.dat and /dev/null differ diff --git a/spec/fixtures/date.dat b/spec/fixtures/date.dat deleted file mode 100644 index 09f50d7..0000000 Binary files a/spec/fixtures/date.dat and /dev/null differ diff --git a/spec/fixtures/date2.dat b/spec/fixtures/date2.dat deleted file mode 100644 index ed8ec49..0000000 Binary files a/spec/fixtures/date2.dat and /dev/null differ diff --git a/spec/fixtures/date2000.dat b/spec/fixtures/date2000.dat deleted file mode 100644 index 64451bb..0000000 Binary files a/spec/fixtures/date2000.dat and /dev/null differ diff --git a/spec/fixtures/dates.dat b/spec/fixtures/dates.dat deleted file mode 100644 index 32d32f0..0000000 Binary files a/spec/fixtures/dates.dat and /dev/null differ diff --git a/spec/fixtures/dates_p924.dat b/spec/fixtures/dates_p924.dat deleted file mode 100644 index 2dff9f6..0000000 Binary files a/spec/fixtures/dates_p924.dat and /dev/null differ diff --git a/spec/fixtures/dates_pg935.dat b/spec/fixtures/dates_pg935.dat deleted file mode 100644 index 2dff9f6..0000000 Binary files a/spec/fixtures/dates_pg935.dat and /dev/null differ diff --git a/spec/fixtures/empty_uuid.dat b/spec/fixtures/empty_uuid.dat deleted file mode 100644 index 077f1ed..0000000 Binary files a/spec/fixtures/empty_uuid.dat and /dev/null differ diff --git a/spec/fixtures/falseclass.dat b/spec/fixtures/falseclass.dat deleted file mode 100644 index b63c2f7..0000000 Binary files a/spec/fixtures/falseclass.dat and /dev/null differ diff --git a/spec/fixtures/float.dat b/spec/fixtures/float.dat deleted file mode 100644 index e57d220..0000000 Binary files a/spec/fixtures/float.dat and /dev/null differ diff --git a/spec/fixtures/geometry_test.dat b/spec/fixtures/geometry_test.dat index 944b6c6..47e71a3 100644 Binary files a/spec/fixtures/geometry_test.dat and b/spec/fixtures/geometry_test.dat differ diff --git a/spec/fixtures/hstore_utf8.dat b/spec/fixtures/hstore_utf8.dat deleted file mode 100644 index 5527304..0000000 Binary files a/spec/fixtures/hstore_utf8.dat and /dev/null differ diff --git a/spec/fixtures/intarray.dat b/spec/fixtures/intarray.dat deleted file mode 100644 index 032d65c..0000000 Binary files a/spec/fixtures/intarray.dat and /dev/null differ diff --git a/spec/fixtures/json.dat b/spec/fixtures/json.dat index 0ae2e9d..9a25048 100644 Binary files a/spec/fixtures/json.dat and b/spec/fixtures/json.dat differ diff --git a/spec/fixtures/json_array.dat b/spec/fixtures/json_array.dat index 5c18fd7..790c55f 100644 Binary files a/spec/fixtures/json_array.dat and b/spec/fixtures/json_array.dat differ diff --git a/spec/fixtures/just_an_array.dat b/spec/fixtures/just_an_array.dat deleted file mode 100644 index 6d0a04c..0000000 Binary files a/spec/fixtures/just_an_array.dat and /dev/null differ diff --git a/spec/fixtures/just_an_array2.dat b/spec/fixtures/just_an_array2.dat deleted file mode 100644 index a45f972..0000000 Binary files a/spec/fixtures/just_an_array2.dat and /dev/null differ diff --git a/spec/fixtures/multiline_hstore.dat b/spec/fixtures/multiline_hstore.dat index ca89a2a..aa4af77 100644 Binary files a/spec/fixtures/multiline_hstore.dat and b/spec/fixtures/multiline_hstore.dat differ diff --git a/spec/fixtures/output.dat b/spec/fixtures/output.dat index 419d5ea..9a25048 100644 Binary files a/spec/fixtures/output.dat and b/spec/fixtures/output.dat differ diff --git a/spec/fixtures/range_test.dat b/spec/fixtures/range_test.dat index 384613b..a0b6af7 100644 Binary files a/spec/fixtures/range_test.dat and b/spec/fixtures/range_test.dat differ diff --git a/spec/fixtures/real.dat b/spec/fixtures/real.dat index c48d1e9..ce3f0f3 100644 Binary files a/spec/fixtures/real.dat and b/spec/fixtures/real.dat differ diff --git a/spec/fixtures/text_array.dat b/spec/fixtures/text_array.dat index 47b0d25..b0af323 100644 Binary files a/spec/fixtures/text_array.dat and b/spec/fixtures/text_array.dat differ diff --git a/spec/fixtures/timestamp.dat b/spec/fixtures/timestamp.dat deleted file mode 100644 index 419d5ea..0000000 Binary files a/spec/fixtures/timestamp.dat and /dev/null differ diff --git a/spec/fixtures/timestamp_9.3.dat b/spec/fixtures/timestamp_9.3.dat deleted file mode 100644 index dceebbd..0000000 Binary files a/spec/fixtures/timestamp_9.3.dat and /dev/null differ diff --git a/spec/fixtures/timestamp_big.dat b/spec/fixtures/timestamp_big.dat deleted file mode 100644 index 3f41aa7..0000000 Binary files a/spec/fixtures/timestamp_big.dat and /dev/null differ diff --git a/spec/fixtures/timestamp_rounding.dat b/spec/fixtures/timestamp_rounding.dat deleted file mode 100644 index 60069e4..0000000 Binary files a/spec/fixtures/timestamp_rounding.dat and /dev/null differ diff --git a/spec/fixtures/trueclass.dat b/spec/fixtures/trueclass.dat deleted file mode 100644 index 019ca75..0000000 Binary files a/spec/fixtures/trueclass.dat and /dev/null differ diff --git a/spec/fixtures/utf8.dat b/spec/fixtures/utf8.dat deleted file mode 100644 index a56ae83..0000000 Binary files a/spec/fixtures/utf8.dat and /dev/null differ diff --git a/spec/fixtures/uuid.dat b/spec/fixtures/uuid.dat deleted file mode 100644 index 0a29399..0000000 Binary files a/spec/fixtures/uuid.dat and /dev/null differ diff --git a/spec/fixtures/uuid_array.dat b/spec/fixtures/uuid_array.dat deleted file mode 100644 index f839f4d..0000000 Binary files a/spec/fixtures/uuid_array.dat and /dev/null differ diff --git a/spec/integration_spec.rb b/spec/integration_spec.rb new file mode 100644 index 0000000..7b476b0 --- /dev/null +++ b/spec/integration_spec.rb @@ -0,0 +1,104 @@ +require File.expand_path(File.dirname(__FILE__) + '/spec_helper') + +# Loading the Dummy Rails App +require File.expand_path('../spec/dummy/config/environment.rb', __dir__) +ENV['RAILS_ROOT'] ||= File.dirname(__FILE__) + '../../../spec/dummy' +require 'rspec/rails' + +describe 'integration' do + it "supports all common fields" do + utf8_stress_test = <<-STRESS +ăѣ𝔠ծềſģȟᎥ𝒋ǩľḿꞑȯ𝘱𝑞𝗋𝘴ȶ𝞄𝜈ψ𝒙𝘆𝚣1234567890!@#$%^&*()-_=+[{]};:'",<.>/?~𝘈Ḇ𝖢𝕯٤ḞԍНǏ𝙅ƘԸⲘ𝙉০Ρ𝗤Ɍ𝓢ȚЦ𝒱Ѡ𝓧ƳȤѧᖯć𝗱ễ𝑓𝙜Ⴙ𝞲𝑗𝒌ļṃʼnо𝞎𝒒ᵲꜱ𝙩ừ𝗏ŵ𝒙𝒚ź1234567890!@#$%^&*()-_=+[{]};:'",<.>/? + STRESS + + factory = RGeo::Geographic.simple_mercator_factory() + polygon = factory.parse_wkt("POLYGON((0.5 0.5,5 0,5 5,0 5,0.5 0.5), (1.5 1,4 3,4 1,1.5 1))") + t1 = Time.new(2002, 10, 31, 12, 5, 1).utc + t2 = Time.new(1985, 10, 31, 3, 4, 5).utc + inet = IPAddr.new("192.168.1.12") + cidr =IPAddr.new("192.168.2.0/24") + + mac_addr = "32:01:16:6d:05:ef" + MyModel.delete_all + columns = [ + :binary, + :bigint, + :boolean, + :date, + :datetime, + :timestamp, + :time, + :decimal, + :float, + :integer, + :text, + :geometry, + :json, + :jsonb, + :inet, + :cidr, + :macaddr + ] + MyModel.copy_from_client(columns) do |csv| + 1000.times do + csv << ["42", 9223372036854775807, true, t1, t1, t1, t1, 4242.42, 4242.42, 42, "this is some text", factory.point(1, 2), { a: 1 }, { a: 1 }, inet, cidr, mac_addr] + csv << ["1", -9223372036854775808, false, t2, t2, t2, t2, -4242.42, 4242.42, -42, utf8_stress_test, polygon, [{ a: 1 }], [{ a: 1 }], inet.to_s, cidr.to_s, "32-01-16-6D:05-EF"] + end + end + + expect(MyModel.count).to be 2000 + + first = MyModel.first + + expect(first.time.hour).to eq(t1.hour) + expect(first.time.min).to eq(t1.min) + expect(first.time.sec).to eq(t1.sec) + + expect(first.attributes.symbolize_keys).to( + include( + binary: "42", + bigint: 9223372036854775807, + decimal: 4242.42, + boolean: true, + date: t1.to_date, + time: Time.new(2000, 1, 1, 12, 5, 1).utc, + timestamp: t1, + datetime: t1, + integer: 42, + text: "this is some text", + geometry: factory.point(1, 2), + json: { "a" => 1 }, + jsonb: { "a" => 1 }, + inet: inet, + cidr: cidr, + macaddr: mac_addr + ) + ) + second = MyModel.second + + expect(second.time.hour).to eq(t2.hour) + expect(second.time.min).to eq(t2.min) + expect(second.time.sec).to eq(t2.sec) + + expect(MyModel.second.attributes.symbolize_keys).to( + include( + binary: "1", + bigint: -9223372036854775808, + decimal: -4242.42, + boolean: false, + time: Time.new(2000, 1, 1, 3, 4, 5).utc, + date: t2.to_date, + datetime: t2, + timestamp: t2, + integer: -42, + text: utf8_stress_test, + geometry: polygon, + json: [{ "a" => 1 }], + jsonb: [{ "a" => 1 }], + inet: inet, + cidr: cidr, + macaddr: mac_addr + ) + ) + end +end diff --git a/spec/multiline_spec.rb b/spec/multiline_spec.rb index 0fec135..59e6d82 100644 --- a/spec/multiline_spec.rb +++ b/spec/multiline_spec.rb @@ -1,17 +1,19 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper') describe 'multiline hstore' do + let(:connection) { + MockConnection::new + } it 'encodes multiline hstore data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [1, { a: 1, b: 2 }] - encoder.add [2, { a: 1, b: 3 }] - encoder.close - io = encoder.get_io - existing_data = filedata('multiline_hstore.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open("spec/fixtures/output.dat", "w:ASCII-8BIT") {|out| out.write(str) } + encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: {0 => nil, 1 => nil}, connection: connection) + encoder.process do + encoder.add [1, { a: 1, b: 2 }] + encoder.add [2, { a: 1, b: 3 }] + end + + existing_data = read_file('multiline_hstore.dat') + str = connection.string + # File.open("spec/fixtures/multiline_hstore.dat", "w:ASCII-8BIT") {|out| out.write(str) } expect(str).to eq existing_data end end diff --git a/spec/setup-db.sh b/spec/setup-db.sh new file mode 100644 index 0000000..a480c49 --- /dev/null +++ b/spec/setup-db.sh @@ -0,0 +1 @@ +DATABASE_URL='postgis://postgres:postgres@localhost:5577' rake db:create db:migrate db:test:prepare \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4dd37a3..837841f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,22 +1,41 @@ $LOAD_PATH.unshift(File.dirname(__FILE__)) $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) +require 'dotenv' +Dotenv.load + require 'rspec' require 'activerecord-copy' require 'rgeo' + RSpec.configure do |config| config.before(:suite) do end end -def filedata(filename) - str = nil - File.open("spec/fixtures/#{filename}", 'r:ASCII-8BIT') do |io| - str = io.read - end - str +def read_file(filename) + File.open("spec/fixtures/#{filename}", 'r:ASCII-8BIT').read end -def fileio(filename) +def open_file(filename) File.open("spec/fixtures/#{filename}", 'r:ASCII-8BIT') end + + +class MockConnection + def initialize + @io = StringIO.new + @io.set_encoding('ASCII-8BIT') + end + + def sync_put_copy_data(buf) + @io.write(buf) + end + + def flush + end + + def string + @io.string + end +end diff --git a/spec/verify_data_formats_spec.rb b/spec/verify_data_formats_spec.rb index 5236584..310039a 100644 --- a/spec/verify_data_formats_spec.rb +++ b/spec/verify_data_formats_spec.rb @@ -3,438 +3,51 @@ require 'date' describe 'generating data' do - it 'encodes hstore data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [1, 'text', { a: 1, b: 'asdf' }] - encoder.close - io = encoder.get_io - existing_data = filedata('3_col_hstore.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes hstore data correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true) - encoder.add [1, 'text', { a: 1, b: 'asdf' }] - encoder.close - io = encoder.get_io - existing_data = filedata('3_col_hstore.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes hstore with utf8 data correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true) - encoder.add [{ test: 'Ekström' }] - encoder.add [{ test: 'Dueñas' }] - encoder.add [{ 'account_first_name' => 'asdfasdf asñas', 'testtesttesttesttesttestest' => '', 'aasdfasdfasdfasdfasdfasdfasdfasdfasfasfasdfs' => '' }] # needed to verify encoding issue - encoder.close - io = encoder.get_io - existing_data = filedata('hstore_utf8.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - it 'encodes TrueClass data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [true] - encoder.close - io = encoder.get_io - existing_data = filedata('trueclass.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes FalseClass data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [false] - encoder.close - io = encoder.get_io - existing_data = filedata('falseclass.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes array data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [%w(hi jim)] - encoder.close - io = encoder.get_io - existing_data = filedata('array_with_two2.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes string array data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [%w(asdfasdfasdfasdf asdfasdfasdfasdfadsfadf 1123423423423)] - encoder.close - io = encoder.get_io - existing_data = filedata('big_str_array.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end + let(:connection) { + MockConnection::new + } it 'encodes text array data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new column_types: { 0 => :text } - encoder.add [['a']] - encoder.close - io = encoder.get_io - existing_data = filedata('text_array.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes string array with big string int' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [['182749082739172']] - encoder.close - io = encoder.get_io - existing_data = filedata('just_an_array2.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes string array data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [%w(asdfasdfasdfasdf asdfasdfasdfasdfadsfadf)] - encoder.close - io = encoder.get_io - existing_data = filedata('big_str_array2.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes array data from tempfile correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true) - encoder.add [1, 'hi', ['hi', 'there', 'rubyist']] - encoder.close - io = encoder.get_io - existing_data = filedata('3_column_array.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes integer array data from tempfile correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true) - encoder.add [[1, 2, 3]] - encoder.close - io = encoder.get_io - existing_data = filedata('intarray.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes date data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Date.parse('1900-12-03')] - encoder.close - io = encoder.get_io - existing_data = filedata('date.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes date data correctly for years > 2000' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Date.parse('2033-01-12')] - encoder.close - io = encoder.get_io - existing_data = filedata('date2000.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes date data correctly in the 70s' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Date.parse('1971-12-11')] - encoder.close - io = encoder.get_io - existing_data = filedata('date2.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes multiple 2015 dates' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Date.parse('2015-04-08'), nil, Date.parse('2015-04-13')] - encoder.close - io = encoder.get_io - existing_data = filedata('dates.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes timestamp data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Time.parse('2013-06-11 15:03:54.62605 UTC')] - encoder.close - io = encoder.get_io - existing_data = filedata('timestamp.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - require 'time' - - # When this timestamp is converted to float it will incorrectly change the - # last digit of usecs to and therefore insert the wrong data into the database - it 'encodes timestamp data correctly without rounding' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Time.parse('2017-07-23 00:02:57.121215 UTC')] - encoder.close - io = encoder.get_io - existing_data = filedata('timestamp_rounding.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/timestamp_rounding.dat', 'w:ASCII-8BIT') { |out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes dates and times in pg 9.2.4' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Date.parse('2015-04-08'), nil, Time.parse('2015-02-13 16:13:57.732772 UTC')] - encoder.close - io = encoder.get_io - existing_data = filedata('dates_p924.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes dates and times in pg 9.3.5' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Date.parse('2015-04-08'), nil, Time.parse('2015-02-13 16:13:57.732772 UTC')] - encoder.close - io = encoder.get_io - existing_data = filedata('dates_pg935.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes timestamp data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Time.parse('2013-06-11 15:03:54.62605 UTC')] - encoder.close - io = encoder.get_io - existing_data = filedata('timestamp.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes big timestamp data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [Time.parse('2014-12-02 16:01:22.437311 UTC')] - encoder.close - io = encoder.get_io - existing_data = filedata('timestamp_9.3.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } + encoder = ActiveRecordCopy::EncodeForCopy.new column_types: { 0 => :text }, connection: connection + encoder.process do + encoder.add [['a']] + end + existing_data = read_file('text_array.dat') + str = connection.string expect(str).to eq existing_data end it 'encodes json hash correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :json }) - encoder.add [{}] - encoder.close - io = encoder.get_io - existing_data = filedata('json.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } + encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :json }, connection: connection) + encoder.process do + encoder.add [{}] + end + existing_data = read_file('json.dat') + str = connection.string + # open_file('json.dat') {|out| out.write(str) } expect(str).to eq existing_data end it 'encodes json array correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :json }) - encoder.add [[]] - encoder.close - io = encoder.get_io - existing_data = filedata('json_array.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes float correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true) - encoder.add [Time.parse('2013-06-11 15:03:54.62605 UTC')] - encoder.close - io = encoder.get_io - existing_data = filedata('timestamp.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes float data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - encoder.add [1_234_567.1234567] - encoder.close - io = encoder.get_io - existing_data = filedata('float.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encode float correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true) - encoder.add [1_234_567.1234567] - encoder.close - io = encoder.get_io - existing_data = filedata('float.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } + encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :json }, connection: connection) + encoder.process do + encoder.add [[]] + end + existing_data = read_file('json_array.dat') + str = connection.string + # File.open('spec/fixtures/json_array.dat', 'w:ASCII-8BIT') {|out| out.write(str) } expect(str).to eq existing_data end it 'encodes real data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new column_types: { 0 => :real } - encoder.add [1_234.1234] - encoder.close - io = encoder.get_io - existing_data = filedata('real.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes uuid correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true, column_types: { 0 => :uuid }) - encoder.add ['e876eef5-a116-4a27-b71f-bac4a1dcd20e'] - encoder.close - io = encoder.get_io - existing_data = filedata('uuid.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes null uuid correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true, column_types: { 1 => :uuid }) - encoder.add ['before2', nil, nil, 123_423_423] - encoder.close - io = encoder.get_io - existing_data = filedata('empty_uuid.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes uuid correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true, column_types: { 0 => :uuid }) - encoder.add [['6272bd7d-adae-44b7-bba1-dca871c2a6fd', '7dc8431f-fcce-4d4d-86f3-6857cba47d38']] - encoder.close - io = encoder.get_io - existing_data = filedata('uuid_array.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes utf8 string correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new - ekstrom = 'Ekström' - encoder.add [ekstrom] - encoder.close - io = encoder.get_io - existing_data = filedata('utf8.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - expect(str).to eq existing_data - end - - it 'encodes bigint as int correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true, column_types: { 0 => :bigint }) - encoder.add [23_372_036_854_775_808, 'test'] - encoder.close - io = encoder.get_io - existing_data = filedata('bigint.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } - expect(str).to eq existing_data - end - - it 'encodes bigint correctly from tempfile' do - encoder = ActiveRecordCopy::EncodeForCopy.new(use_tempfile: true, column_types: { 0 => :bigint }) - encoder.add %w(23372036854775808 test) - encoder.close - io = encoder.get_io - existing_data = filedata('bigint.dat') - str = io.read - expect(io.class.name).to eq 'Tempfile' - str.force_encoding('ASCII-8BIT') - # File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } + encoder = ActiveRecordCopy::EncodeForCopy.new column_types: { 0 => :real }, connection: connection + encoder.process do + encoder.add [1_234.1234] + end + existing_data = read_file('real.dat') + str = connection.string + # File.open('spec/fixtures/real.dat', 'w:ASCII-8BIT') {|out| out.write(str) } expect(str).to eq existing_data end @@ -442,36 +55,35 @@ # INSERT INTO test VALUES ('[12, 14)', '[223372033854775802, 223372033854775810)', '[12.5,13.88211]', '[2010-01-01 15:20, 2010-01-01 15:30)', '[2018-05-24 00:00:00+00,)', '[2018-05-24,)'); # \copy test TO range_test.dat WITH (FORMAT BINARY); it 'encodes range data correctly' do - encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :int4range, 1 => :int8range, 2 => :numrange, 3 => :tsrange, 4 => :tstzrange, 5 => :daterange }) - encoder.add([ - 12...14, - 223372033854775802...223372033854775810, - 12.5..13.88211, - Time.parse('2010-01-01 15:20+00')...Time.parse('2010-01-01 15:30+00'), - Time.parse('2018-05-24 00:00:00+00')...Float::INFINITY, - Date.parse('2018-05-24')...Float::INFINITY - ]) - encoder.close - io = encoder.get_io - existing_data = filedata('range_test.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') - #File.open('spec/fixtures/output.dat', 'w:ASCII-8BIT') {|out| out.write(str) } + encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :int4range, 1 => :int8range, 2 => :numrange, 3 => :tsrange, 4 => :tstzrange, 5 => :daterange }, connection: connection) + encoder.process do + encoder.add( + [ + 12...14, + 223372033854775802...223372033854775810, + 12.5..13.88211, + Time.parse('2010-01-01 15:20+00')...Time.parse('2010-01-01 15:30+00'), + Time.parse('2018-05-24 00:00:00+00')...Float::INFINITY, + Date.parse('2018-05-24')...Float::INFINITY + ] + ) + end + existing_data = read_file('range_test.dat') + str = connection.string + # File.open('spec/fixtures/range_test.dat', 'w:ASCII-8BIT') {|out| out.write(str) } expect(str).to eq existing_data end it 'encodes geometry correctly' do factory = RGeo::Geographic.simple_mercator_factory() point = factory.point(0, 0) - encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :geometry }) - encoder.add([point]) - encoder.close - io = encoder.get_io - existing_data = filedata('geometry_test.dat') - str = io.read - expect(io.class.name).to eq 'StringIO' - str.force_encoding('ASCII-8BIT') + encoder = ActiveRecordCopy::EncodeForCopy.new(column_types: { 0 => :geometry }, connection: connection) + encoder.process do + encoder.add([point]) + end + + existing_data = read_file('geometry_test.dat') + str = connection.string # File.open('spec/fixtures/geometry_test.dat', 'w:ASCII-8BIT') {|out| out.write(str) } expect(str).to eq existing_data end diff --git a/spec/verify_decoder_spec.rb b/spec/verify_decoder_spec.rb deleted file mode 100644 index b56911e..0000000 --- a/spec/verify_decoder_spec.rb +++ /dev/null @@ -1,263 +0,0 @@ -# encoding: utf-8 -require File.expand_path(File.dirname(__FILE__) + '/spec_helper') -require 'time' - -describe 'parsing data' do - it 'walks through each line and stop' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('hstore_utf8.dat'), - column_types: { 0 => :hstore } - ) - lines = [] - decoder.each do |l| - lines << l - end - expect(lines).to eq [ - [{ 'test' => 'Ekström' }], - [{ 'test' => 'Dueñas' }], - [{ 'account_first_name' => 'asdfasdf asñas', 'testtesttesttesttesttestest' => '', 'aasdfasdfasdfasdfasdfasdfasdfasdfasfasfasdfs' => '' }] - ] - end - - it 'handles getting called after running out of data' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('3_col_hstore.dat'), - column_types: { 0 => :int, 1 => :string, 2 => :hstore } - ) - r = decoder.read_line - expect(r).to eq [1, 'text', { 'a' => '1', 'b' => 'asdf' }] - expect(decoder.read_line).to be_nil - expect(decoder.read_line).to be_nil - end - - it 'encodes hstore data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('3_col_hstore.dat'), - column_types: { 0 => :int, 1 => :string, 2 => :hstore } - ) - r = decoder.read_line - expect(r).to eq [1, 'text', { 'a' => '1', 'b' => 'asdf' }] - end - - it 'returns nil if past data' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('3_col_hstore.dat'), - column_types: { 0 => :int, 1 => :string, 2 => :hstore } - ) - r = decoder.read_line - expect(r).to eq [1, 'text', { 'a' => '1', 'b' => 'asdf' }] - r = decoder.read_line - expect(r).to eq nil - end - - it 'encodes hstore with utf8 data correctly from tempfile' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('hstore_utf8.dat'), - column_types: { 0 => :hstore } - ) - r = decoder.read_line - expect(r).to eq [{ 'test' => 'Ekström' }] - r = decoder.read_line - expect(r).to eq [{ 'test' => 'Dueñas' }] - r = decoder.read_line - expect(r).to eq [{ 'account_first_name' => 'asdfasdf asñas', 'testtesttesttesttesttestest' => '', 'aasdfasdfasdfasdfasdfasdfasdfasdfasfasfasdfs' => '' }] - end - - it 'encodes TrueClass data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('trueclass.dat'), - column_types: { 0 => :boolean } - ) - r = decoder.read_line - expect(r).to eq [true] - end - - it 'encodes FalseClass data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('falseclass.dat'), - column_types: { 0 => :boolean } - ) - r = decoder.read_line - expect(r).to eq [false] - end - - it 'encodes array data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('array_with_two2.dat'), - column_types: { 0 => :array } - ) - r = decoder.read_line - expect(r).to eq [%w(hi jim)] - end - - it 'encodes string array data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('big_str_array.dat'), - column_types: { 0 => :array } - ) - r = decoder.read_line - expect(r).to eq [%w(asdfasdfasdfasdf asdfasdfasdfasdfadsfadf 1123423423423)] - end - - it 'encodes string array with big string int' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('just_an_array2.dat'), - column_types: { 0 => :array } - ) - r = decoder.read_line - expect(r).to eq [['182749082739172']] - end - - it 'encodes string array data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('big_str_array2.dat'), - column_types: { 0 => :array } - ) - r = decoder.read_line - expect(r).to eq [%w(asdfasdfasdfasdf asdfasdfasdfasdfadsfadf)] - end - - it 'encodes array data from tempfile correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('3_column_array.dat'), - column_types: { 0 => :integer, 1 => :string, 2 => :array } - ) - r = decoder.read_line - expect(r).to eq [1, 'hi', ['hi', 'there', 'rubyist']] - end - - it 'encodes integer array data from tempfile correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('intarray.dat'), - column_types: { 0 => :array } - ) - r = decoder.read_line - expect(r).to eq [[1, 2, 3]] - end - - it 'encodes old date data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('date.dat'), - column_types: { 0 => :date } - ) - r = decoder.read_line - expect(r).to eq [Date.parse('1900-12-03')] - end - - it 'encodes date data correctly for years > 2000' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('date2000.dat'), - column_types: { 0 => :date } - ) - r = decoder.read_line - expect(r).to eq [Date.parse('2033-01-12')] - end - - it 'encodes date data correctly in the 70s' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('date2.dat'), - column_types: { 0 => :date } - ) - r = decoder.read_line - expect(r).to eq [Date.parse('1971-12-11')] - end - - it 'encodes multiple 2015 dates' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('dates.dat'), - column_types: { 0 => :date, 1 => :date, 2 => :date } - ) - r = decoder.read_line - expect(r).to eq [Date.parse('2015-04-08'), nil, Date.parse('2015-04-13')] - end - - it 'encodes timestamp data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('timestamp.dat'), - column_types: { 0 => :time } - ) - r = decoder.read_line - expect(r.map(&:to_f)).to eq [Time.parse('2013-06-11 15:03:54.62605 UTC')].map(&:to_f) - end - - it 'encodes dates and times in pg 9.2.4' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('dates_p924.dat'), - column_types: { 0 => :date, 2 => :time } - ) - r = decoder.read_line - expect(r.map { |f| f.is_a?(Time) ? f.to_f : f }).to eq [Date.parse('2015-04-08'), nil, Time.parse('2015-02-13 16:13:57.732772 UTC')].map { |f| f.is_a?(Time) ? f.to_f : f } - end - - it 'encodes dates and times in pg 9.3.5' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('dates_pg935.dat'), - column_types: { 0 => :date, 2 => :time } - ) - r = decoder.read_line - expect(r.map { |f| f.is_a?(Time) ? f.to_f : f }).to eq [Date.parse('2015-04-08'), nil, Time.parse('2015-02-13 16:13:57.732772 UTC')].map { |f| f.is_a?(Time) ? f.to_f : f } - end - - it 'encodes big timestamp data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('timestamp_9.3.dat'), - column_types: { 0 => :time } - ) - r = decoder.read_line - expect(r.map { |f| f.is_a?(Time) ? f.to_f : f }).to eq [Time.parse('2014-12-02 16:01:22.437311 UTC')].map { |f| f.is_a?(Time) ? f.to_f : f } - end - - it 'encodes float data correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('float.dat'), - column_types: { 0 => :float } - ) - r = decoder.read_line - expect(r).to eq [1_234_567.1234567] - end - - it 'encodes uuid correctly from tempfile' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('uuid.dat'), - column_types: { 0 => :uuid } - ) - r = decoder.read_line - expect(r).to eq ['e876eef5-a116-4a27-b71f-bac4a1dcd20e'] - end - - it 'encodes null uuid correctly from tempfile' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('empty_uuid.dat'), - column_types: { 0 => :string, 1 => :uuid, 2 => :uuid, 3 => :integer } - ) - r = decoder.read_line - expect(r).to eq ['before2', nil, nil, 123_423_423] - end - - it 'encodes uuid correctly from tempfile' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('uuid_array.dat'), - column_types: { 0 => :array } - ) - r = decoder.read_line - expect(r).to eq [['6272bd7d-adae-44b7-bba1-dca871c2a6fd', '7dc8431f-fcce-4d4d-86f3-6857cba47d38']] - end - - it 'encodes utf8 string correctly' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('utf8.dat'), - column_types: { 0 => :string } - ) - r = decoder.read_line - expect(r).to eq ['Ekström'] - end - - it 'encodes bigint as int correctly from tempfile' do - decoder = ActiveRecordCopy::Decoder.new( - io: fileio('bigint.dat'), - column_types: { 0 => :bigint, 1 => :string } - ) - r = decoder.read_line - expect(r).to eq [23_372_036_854_775_808, 'test'] - end -end diff --git a/start-local-db.sh b/start-local-db.sh new file mode 100755 index 0000000..ec8a45a --- /dev/null +++ b/start-local-db.sh @@ -0,0 +1 @@ +docker-compose up db \ No newline at end of file