diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..931d082 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ruby: ["3.2", "3.3", "3.4", "4.0"] + + steps: + - uses: actions/checkout@v4 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + + - run: bundle exec rspec + + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.4" + bundler-cache: true + + - run: bundle exec rubocop diff --git a/.rubocop.yml b/.rubocop.yml index 67ab06a..2416daa 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,4 +1,8 @@ -Documentation: +AllCops: + TargetRubyVersion: 3.2 + NewCops: enable + +Style/Documentation: Enabled: false Style/StringLiterals: @@ -11,5 +15,12 @@ Metrics/BlockLength: Exclude: - spec/**/* -Metrics/LineLength: +Layout/LineLength: Max: 120 + +Gemspec/DevelopmentDependencies: + EnforcedStyle: gemspec + +Metrics/ClassLength: + Exclude: + - lib/deepl_diff/tokenizer.rb diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..7636e75 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +4.0.5 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 26802db..0000000 --- a/.travis.yml +++ /dev/null @@ -1,12 +0,0 @@ -language: ruby -sudo: false -rvm: - - 2.3.3 -cache: bundler - -addons: - code_climate: - repo_token: "49b1afe0298c521f3d73377db0d20d1d4b9749ad533126bf3ed105fe9612eb0e" - -after_success: - - bundle exec codeclimate-test-reporter diff --git a/deepl_diff.gemspec b/deepl_diff.gemspec index 6ad476c..0164398 100644 --- a/deepl_diff.gemspec +++ b/deepl_diff.gemspec @@ -4,7 +4,7 @@ lib = File.expand_path("lib", __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require "deepl_diff/version" -# rubocop:disable Metrics/BlockLength +# rubocop:disable-next Metrics/BlockLength Gem::Specification.new do |spec| spec.name = "deepl_diff" spec.version = DeepLDiff::VERSION @@ -21,12 +21,16 @@ DeepL API wrapper for Ruby which helps to translate only changes between revisions of long texts. ) spec.homepage = "https://github.com/Halvanhelv/deepl_diff" + spec.required_ruby_version = ">= 3.2" if spec.respond_to?(:metadata) spec.metadata["allowed_push_host"] = "https://rubygems.org" + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = spec.homepage + spec.metadata["rubygems_mfa_required"] = "true" else raise "RubyGems 2.0 or newer is required to protect against " \ - "public gem pushes." + "public gem pushes." end spec.files = `git ls-files -z`.split("\x0").reject do |f| @@ -36,20 +40,17 @@ between revisions of long texts. spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] - spec.add_development_dependency "bundler", "~> 1.14" - spec.add_development_dependency "codeclimate-test-reporter", "~> 1.0.0" - spec.add_development_dependency "rake", "~> 10.0" - spec.add_development_dependency "rspec", "~> 3.0" - spec.add_development_dependency "rubocop" - spec.add_development_dependency "simplecov" - - spec.add_dependency "connection_pool" - spec.add_dependency "deepl-rb" - spec.add_dependency "dry-initializer" - spec.add_dependency "ox" - spec.add_dependency "punkt-segmenter" - spec.add_dependency "ratelimit" - spec.add_dependency "redis" - spec.add_dependency "redis-namespace" + spec.add_development_dependency "rake", "~> 13.0" + spec.add_development_dependency "rspec", "~> 3.13" + spec.add_development_dependency "rubocop", "~> 1.90" + spec.add_development_dependency "simplecov", "~> 1.2" + + spec.add_dependency "connection_pool", ">= 2.4", "< 4.0" + spec.add_dependency "deepl-rb", "~> 3.9" + spec.add_dependency "dry-initializer", "~> 3.2" + spec.add_dependency "ox", "~> 2.14" + spec.add_dependency "punkt-segmenter", "~> 0.9" + spec.add_dependency "ratelimit", "~> 1.1" + spec.add_dependency "redis", ">= 5.0", "< 7.0" + spec.add_dependency "redis-namespace", "~> 1.11" end -# rubocop:enable Metrics/BlockLength diff --git a/lib/deepl_diff.rb b/lib/deepl_diff.rb index 47ec036..8c6d75e 100644 --- a/lib/deepl_diff.rb +++ b/lib/deepl_diff.rb @@ -1,5 +1,10 @@ # frozen_string_literal: true +require "cgi/escape" +require "digest/md5" +require "forwardable" +require "stringio" + require "ox" require "punkt-segmenter" require "dry/initializer" @@ -19,8 +24,8 @@ module DeepLDiff class << self attr_accessor :api, :cache_store, :rate_limiter - def translate(*args) - Request.new(*args).call + def translate(*) + Request.new(*).call end end diff --git a/lib/deepl_diff/chunker.rb b/lib/deepl_diff/chunker.rb index 607e133..c9cc6e4 100644 --- a/lib/deepl_diff/chunker.rb +++ b/lib/deepl_diff/chunker.rb @@ -5,14 +5,17 @@ class DeepLDiff::Chunker class Error < StandardError; end - Chunk = Struct.new(:values, :size) + Chunk = Struct.new(:texts, :bytesize) + + MAX_CHUNK_SIZE = 1700 + COUNT_LIMIT = 300 param :values option :limit, default: proc { MAX_CHUNK_SIZE } option :count_limit, default: proc { COUNT_LIMIT } def call - chunks.map(&:values) + chunks.map(&:texts) end def chunks @@ -34,8 +37,8 @@ def chunks def next_chunk?(tail, value) tail.nil? || - (size(value) + tail.size > limit) || - tail.values.size > count_limit + (size(value) + tail.bytesize > limit) || + tail.texts.size > count_limit end def size(text) @@ -43,14 +46,11 @@ def size(text) end def update_chunk(chunk, value) - chunk.values << value - chunk.size = chunk.size + value.size + chunk.texts << value + chunk.bytesize += value.size end def validate_value_size(value) raise Error, "Too long part #{value.size} > #{limit}" if value.size > limit end - - MAX_CHUNK_SIZE = 1700 - COUNT_LIMIT = 300 end diff --git a/lib/deepl_diff/linearizer.rb b/lib/deepl_diff/linearizer.rb index b0e8f61..1b9e189 100644 --- a/lib/deepl_diff/linearizer.rb +++ b/lib/deepl_diff/linearizer.rb @@ -5,7 +5,7 @@ class << self def linearize(struct, array = []) case struct when Hash - struct.each { |_k, v| linearize(v, array) } + struct.each_value { |v| linearize(v, array) } when Array struct.each { |v| linearize(v, array) } else diff --git a/lib/deepl_diff/request.rb b/lib/deepl_diff/request.rb index fff6fdd..87b461f 100644 --- a/lib/deepl_diff/request.rb +++ b/lib/deepl_diff/request.rb @@ -99,13 +99,13 @@ def chunks_translated # Restores indexes for translated tokens # => { ..., "1_1" => "Horoshiy", 1_3 => "Malchik", ... } def text_tokens_translated - @text_tokens_texts_translated ||= + @text_tokens_translated ||= restore(text_tokens, chunks_translated.flatten) end # Restores tokens translated + adds same spacing as in source token # => [[..., [ "Horoshiy", :text ], ...]] - # rubocop:disable Metrics/AbcSize + # rubocop:disable-next Metrics/AbcSize def tokens_translated @tokens_translated ||= tokens.dup.tap do |tokens| text_tokens_translated.each do |index, value| @@ -115,7 +115,6 @@ def tokens_translated end end end - # rubocop:enable Metrics/AbcSize def restore_spacing(source_value, value) DeepLDiff::Spacing.restore(source_value, value) @@ -136,7 +135,7 @@ def translation def call_api(values) check_rate_limit(values) - [api.translate(values, from, to, **options)].flatten.map(&:text) + [api.translate(values, from, to, options)].flatten.map(&:text) end def cache diff --git a/lib/deepl_diff/tokenizer.rb b/lib/deepl_diff/tokenizer.rb index 8c8c607..a609ec3 100644 --- a/lib/deepl_diff/tokenizer.rb +++ b/lib/deepl_diff/tokenizer.rb @@ -1,6 +1,12 @@ # frozen_string_literal: true -class DeepLDiff::Tokenizer < ::Ox::Sax +class DeepLDiff::Tokenizer < Ox::Sax + SKIP = %i[script style].freeze + INNER_SPANS = %i[notranslate span end_span end_notranslate].freeze + HTML_OPTIONS = { smart: true, skip: :skip_none }.freeze + + # Ox::Sax provides no initializer to chain to. + # rubocop:disable-next Lint/MissingSuper def initialize(source) @pos = nil @source = source @@ -38,7 +44,7 @@ def text(value) return if value == "" @sequence << (SKIP.include?(@context.last) ? :markup : :text) - @indicies << @pos - 1 + @indicies << (@pos - 1) end def tokens @@ -69,7 +75,7 @@ def make_sentences_from_last_token(tokens) tokens.concat(sentences(tokens.pop[0])) if tokens.last[1] == :text end - # rubocop: disable Metrics/MethodLength + # rubocop: disable-next Metrics/MethodLength def sentences(value) return [] if value.strip.empty? @@ -87,7 +93,6 @@ def sentences(value) [value[left..right], :text] end end - # rubocop:enable Metrics/MethodLength # Whether the sequence is between `:notranslate` and `:end_notranslate` def notranslate? @@ -129,7 +134,7 @@ def start_markup(name) else :markup end) - @indicies << @pos - 1 + @indicies << (@pos - 1) end def end_markup(name) @@ -139,7 +144,7 @@ def end_markup(name) else :markup end) - @indicies << @pos - 1 unless @pos == @source.bytesize + @indicies << (@pos - 1) unless @pos == @source.bytesize end class << self @@ -152,8 +157,4 @@ def tokenize(value) tokenizer.tokens end end - - SKIP = %i[script style].freeze - INNER_SPANS = %i[notranslate span end_span end_notranslate].freeze - HTML_OPTIONS = { smart: true, skip: :skip_none }.freeze end diff --git a/lib/deepl_diff/version.rb b/lib/deepl_diff/version.rb index 976f8f8..8608e35 100644 --- a/lib/deepl_diff/version.rb +++ b/lib/deepl_diff/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module DeepLDiff - VERSION = "1.0.2" + VERSION = "1.1.0" end diff --git a/spec/deepl_diff/request_spec.rb b/spec/deepl_diff/request_spec.rb index b156d2c..1b4cd55 100644 --- a/spec/deepl_diff/request_spec.rb +++ b/spec/deepl_diff/request_spec.rb @@ -5,9 +5,13 @@ RSpec.describe DeepLDiff::Request do subject { described_class.new(values, options).call } + # OpenStruct is no longer available by default, and the API response only + # needs to expose #text. + translation = Struct.new(:text) + let(:api) { double("API") } let(:cache_store) { double("Cache store") } - let(:api_response_wrap) { api_response.map { |v| OpenStruct.new(text: v) } } + let(:api_response_wrap) { api_response.map { |v| translation.new(v) } } let(:cache_response) { nil } before do diff --git a/spec/deepl_diff/tokenizer_spec.rb b/spec/deepl_diff/tokenizer_spec.rb index 4b058ca..769151d 100644 --- a/spec/deepl_diff/tokenizer_spec.rb +++ b/spec/deepl_diff/tokenizer_spec.rb @@ -134,8 +134,8 @@ context "notranslate inside another notranslate" do let(:source) do "foo" \ - "barbaz" \ - "" + "barbaz" \ + "" end let(:tokens) do @@ -166,7 +166,7 @@ context "with tag" do let(:source) do "Hey!
Look!" \ - "" + "" end let(:tokens) do @@ -184,7 +184,7 @@ context "bizarre sentences" do let(:source) do "Набор «Солнечная механика» от 4М — это 6 экспериментов." \ - "\n\nЮному изобретателю предстоит воочию посмотреть на чудеса." + "\n\nЮному изобретателю предстоит воочию посмотреть на чудеса." end let(:tokens) do