From 2ed0576884496a2afb52ac2bcdb3b844d9790c36 Mon Sep 17 00:00:00 2001 From: Mario Gomez Date: Mon, 30 Mar 2020 19:17:49 +0300 Subject: [PATCH 1/7] Added PoC for integration to scrapper --- workers/sms_request_worker.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/workers/sms_request_worker.rb b/workers/sms_request_worker.rb index 7ec942a..31732f7 100644 --- a/workers/sms_request_worker.rb +++ b/workers/sms_request_worker.rb @@ -1,3 +1,4 @@ +require 'net/http' require "sidekiq" require "rack/env" require "sinatra/activerecord" @@ -13,9 +14,17 @@ class SmsRequestWorker def perform(sms_request_id) puts "::::::::::::::::::::::::::::::::::::::::::::::::::: Worker doing somthing \n" sms_request=SmsRequest.find(sms_request_id) + # TODO: Add endpoint as parameter + uri = URI('https://sms-scrapper.rover.quenecesito.org/sms/'+sms_request.dui) + # TODO: Add timeout or cancel + Net::HTTP.start(uri.host,uri.port, + :use_ssl => uri.scheme == 'https') do |http| + request = Net::HTTP::Get.new uri + response = http.request request + end - # GET MESSAGE - message = "Buxos ya llegan los sms, solo falta integrar con scraping XD" + # TODO: Process response or enqueue again in case of error + message = response.read_body if sms_request resp=TwilioSms.send_sms(sms_request.phone, message) @@ -25,4 +34,4 @@ def perform(sms_request_id) p "The work is done: #{sms_request.inspect}" end end -end \ No newline at end of file +end From 0f2c35ab6154b3911c743cd9ca95b85ee77a3dc8 Mon Sep 17 00:00:00 2001 From: Mario Gomez Date: Mon, 30 Mar 2020 19:24:03 +0300 Subject: [PATCH 2/7] Added validation for response code --- workers/sms_request_worker.rb | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/workers/sms_request_worker.rb b/workers/sms_request_worker.rb index 31732f7..d8edcbd 100644 --- a/workers/sms_request_worker.rb +++ b/workers/sms_request_worker.rb @@ -14,24 +14,31 @@ class SmsRequestWorker def perform(sms_request_id) puts "::::::::::::::::::::::::::::::::::::::::::::::::::: Worker doing somthing \n" sms_request=SmsRequest.find(sms_request_id) - # TODO: Add endpoint as parameter - uri = URI('https://sms-scrapper.rover.quenecesito.org/sms/'+sms_request.dui) - # TODO: Add timeout or cancel - Net::HTTP.start(uri.host,uri.port, + + if sms_request + # TODO: Add endpoint as parameter + uri = URI('https://sms-scrapper.rover.quenecesito.org/sms/'+sms_request.dui) + + # TODO: Add timeout or cancel + Net::HTTP.start(uri.host,uri.port, :use_ssl => uri.scheme == 'https') do |http| - request = Net::HTTP::Get.new uri - response = http.request request - end + request = Net::HTTP::Get.new uri + response = http.request request + end - # TODO: Process response or enqueue again in case of error - message = response.read_body + if response.code == 200 + # TODO: Process response or enqueue again in case of error + message = response.read_body - if sms_request - resp=TwilioSms.send_sms(sms_request.phone, message) + resp=TwilioSms.send_sms(sms_request.phone, message) + + sms_request.update_column(:status, 1) + sms_request.update_column(:last_error, resp) - sms_request.update_column(:status, 1) - sms_request.update_column(:last_error, resp) - p "The work is done: #{sms_request.inspect}" + p "The work is done: #{sms_request.inspect}" + else + p "Request failed: #{sms_request.inspect}" + end end end end From 89d46d2752b063e065b0bfefff4fffe5eedbfb34 Mon Sep 17 00:00:00 2001 From: Rodrigo Puente Date: Mon, 30 Mar 2020 10:33:50 -0600 Subject: [PATCH 3/7] some todos done --- .env | 3 ++- .ruby-version | 1 + workers/sms_request_worker.rb | 27 +++++++++++++++++++-------- 3 files changed, 22 insertions(+), 9 deletions(-) create mode 100644 .ruby-version diff --git a/.env b/.env index 3003456..c35568e 100644 --- a/.env +++ b/.env @@ -7,4 +7,5 @@ REDIS_PORT=6379 REDIS_PASSWORD= TWILIO_ACCOUNT_SID=a TWILIO_ACCOUNT_TOKEN=b -TWILIO_PHONE_NUMBER=c \ No newline at end of file +TWILIO_PHONE_NUMBER=c +SCRAPPER_SERVICE=https://sms-scrapper.rover.quenecesito.org/sms \ No newline at end of file diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..24ba9a3 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +2.7.0 diff --git a/workers/sms_request_worker.rb b/workers/sms_request_worker.rb index 31732f7..15eebfd 100644 --- a/workers/sms_request_worker.rb +++ b/workers/sms_request_worker.rb @@ -13,14 +13,25 @@ class SmsRequestWorker def perform(sms_request_id) puts "::::::::::::::::::::::::::::::::::::::::::::::::::: Worker doing somthing \n" - sms_request=SmsRequest.find(sms_request_id) - # TODO: Add endpoint as parameter - uri = URI('https://sms-scrapper.rover.quenecesito.org/sms/'+sms_request.dui) - # TODO: Add timeout or cancel - Net::HTTP.start(uri.host,uri.port, - :use_ssl => uri.scheme == 'https') do |http| - request = Net::HTTP::Get.new uri - response = http.request request + sms_request = SmsRequest.find(sms_request_id) + + uri = URI("#{ENV['SCRAPPER_SERVICE']}/#{sms_request.dui}") + + begin + Net::HTTP.start( + uri.host, + uri.port, + :use_ssl => uri.scheme == 'https', + :read_timeout => 30, + :open_timeout => 30 + ) do |http| + request = Net::HTTP::Get.new uri + response = http.request request + end + rescue Net::ReadTimeout => exception + STDERR.puts "#{host}:#{port} ReadTimeout error" + rescue Net::OpenTimeout => exception + STDERR.puts "#{host}:#{port} OpenTimeout error" end # TODO: Process response or enqueue again in case of error From 471d65795b7bdbe38c94fcfee36f10e298d9bd2e Mon Sep 17 00:00:00 2001 From: Mario Gomez Date: Mon, 30 Mar 2020 19:42:10 +0300 Subject: [PATCH 4/7] Added timeout for HTTP resquest --- workers/sms_request_worker.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/workers/sms_request_worker.rb b/workers/sms_request_worker.rb index d8edcbd..41bd0c9 100644 --- a/workers/sms_request_worker.rb +++ b/workers/sms_request_worker.rb @@ -19,9 +19,10 @@ def perform(sms_request_id) # TODO: Add endpoint as parameter uri = URI('https://sms-scrapper.rover.quenecesito.org/sms/'+sms_request.dui) - # TODO: Add timeout or cancel + # TODO: Add timeout as parameter Net::HTTP.start(uri.host,uri.port, - :use_ssl => uri.scheme == 'https') do |http| + :use_ssl => uri.scheme == 'https', + :read_timeout => 10) do |http| request = Net::HTTP::Get.new uri response = http.request request end From 8124ce0695c88e0ba9ae878b258cf818803923fb Mon Sep 17 00:00:00 2001 From: Rodrigo Puente Date: Mon, 30 Mar 2020 10:53:59 -0600 Subject: [PATCH 5/7] fixed if/begin --- workers/sms_request_worker.rb | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/workers/sms_request_worker.rb b/workers/sms_request_worker.rb index 6b0ab6d..bbcdfcf 100644 --- a/workers/sms_request_worker.rb +++ b/workers/sms_request_worker.rb @@ -17,16 +17,17 @@ def perform(sms_request_id) sms_request = SmsRequest.find(sms_request_id) if sms_request - uri = URI("#{ENV['SCRAPPER_SERVICE']}/#{sms_request.dui}") + uri = URI("#{ENV['SCRAPPER_ENDPOINT']}/#{sms_request.dui}") - # TODO: Add timeout as parameter - Net::HTTP.start(uri.host,uri.port, - :use_ssl => uri.scheme == 'https', - :read_timeout => 10, - :open_timeout => 10) do |http| - request = Net::HTTP::Get.new uri - response = http.request request - end + begin + # TODO: Add timeout as parameter + Net::HTTP.start(uri.host,uri.port, + :use_ssl => uri.scheme == 'https', + :read_timeout => 10, + :open_timeout => 10) do |http| + request = Net::HTTP::Get.new uri + response = http.request request + end if response.code == 200 # TODO: Process response or enqueue again in case of error @@ -46,5 +47,6 @@ def perform(sms_request_id) rescue Net::OpenTimeout => exception STDERR.puts "OpenTimeout error" end + end end end From 57894f56260940b9337828ba94171d3ef1a0d70d Mon Sep 17 00:00:00 2001 From: Mario Gomez Date: Mon, 30 Mar 2020 20:26:07 +0300 Subject: [PATCH 6/7] Added failure codes. --- workers/sms_request_worker.rb | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/workers/sms_request_worker.rb b/workers/sms_request_worker.rb index 41bd0c9..a0bcc6d 100644 --- a/workers/sms_request_worker.rb +++ b/workers/sms_request_worker.rb @@ -1,4 +1,5 @@ -require 'net/http' +require "net/http" +require "json" require "sidekiq" require "rack/env" require "sinatra/activerecord" @@ -12,24 +13,30 @@ class SmsRequestWorker include Sidekiq::Worker def perform(sms_request_id) - puts "::::::::::::::::::::::::::::::::::::::::::::::::::: Worker doing somthing \n" + puts "::::::::::::::::::::::::::::::::::::::::::::::::::: Worker doing something \n" sms_request=SmsRequest.find(sms_request_id) - if sms_request - # TODO: Add endpoint as parameter - uri = URI('https://sms-scrapper.rover.quenecesito.org/sms/'+sms_request.dui) - - # TODO: Add timeout as parameter + if sms_request + uri = URI(ENV['SCRAPPER_ENDPOINT']+sms_request.dui) + timeout = ENV['SCRAPPER_REQUEST_TIMEOUT'].to_i + Net::HTTP.start(uri.host,uri.port, :use_ssl => uri.scheme == 'https', - :read_timeout => 10) do |http| + :read_timeout => timeout) do |http| request = Net::HTTP::Get.new uri response = http.request request end if response.code == 200 - # TODO: Process response or enqueue again in case of error - message = response.read_body + response = JSON.parse(response.read_body) + + message = "" + # Re-escribiendo el mensaje + if response["success"] == true + message = "Usted es beneficiario de los $300USD" + else + message = "Usted no es beneficiario. Llame al 2565-5555 si aplica al criterio de seleccion" + end resp=TwilioSms.send_sms(sms_request.phone, message) @@ -38,7 +45,7 @@ def perform(sms_request_id) p "The work is done: #{sms_request.inspect}" else - p "Request failed: #{sms_request.inspect}" + p "Request failed by HTTP error: "+response.code.to_s end end end From 846c5c87b090ab2e88018a27cf45a4c4320fb06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roberto=20Pe=C3=B1a?= Date: Tue, 31 Mar 2020 13:14:20 -0500 Subject: [PATCH 7/7] =?UTF-8?q?[add]=20Configuraci=C3=B3n=20para=20enviar?= =?UTF-8?q?=20mensages=20a=20Dead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit El objetivo es sacar de la cola los posibles "mensajes enveneados", puede pensarse en hacer un handler para sacarlo de la tabla, en caso que el error sea temporal. --- .env | 3 ++- workers/sms_request_worker.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.env b/.env index c35568e..26b0b7e 100644 --- a/.env +++ b/.env @@ -8,4 +8,5 @@ REDIS_PASSWORD= TWILIO_ACCOUNT_SID=a TWILIO_ACCOUNT_TOKEN=b TWILIO_PHONE_NUMBER=c -SCRAPPER_SERVICE=https://sms-scrapper.rover.quenecesito.org/sms \ No newline at end of file +SCRAPPER_SERVICE=https://sms-scrapper.rover.quenecesito.org/sms +RETRY_COUNT=3 \ No newline at end of file diff --git a/workers/sms_request_worker.rb b/workers/sms_request_worker.rb index ecbed20..07801c8 100644 --- a/workers/sms_request_worker.rb +++ b/workers/sms_request_worker.rb @@ -11,6 +11,7 @@ end class SmsRequestWorker include Sidekiq::Worker + sidekiq_options :retry => ENV['RETRY_COUNT'].to_i def perform(sms_request_id) puts "::::::::::::::::::::::::::::::::::::::::::::::::::: Worker doing something \n"