Update weather API to Tomorrowio (#689)

* Update weather API

* Use postal code

Postal code is non nullable in the DB

* Update weather retriever to use dig

* Standardrb tool fixes

* Ignore .DS_Store files

* Partial change to VCR

* Pin tomorrowio dependency

* Fix Gemfile.lock

* Fix bugs and tests

* Add local test instructions

.gitignore

@@ -9,4 +9,6 @@ dump.rdb
 
 .idea/*
 .bundle
-frontend/.env
+frontend/.env
+
+.DS_Store

README.md

@@ -108,6 +108,11 @@ docker compose up
 rake run
 ```
 
+## Running tests locally
+1. Run `docker compose build backend` to ensure the latest backend is being run
+2. To run all tests run `script/backend rspec spec spec`, or you can run a specific test suite such as `script/backend rspec spec spec/services/weather_retriever_spec.rb `
+3. Debugging tip: in Ruby code you can add a line that says `debugger` and rspec will automatically break on that line and give you an interactive Ruby shell
+
 ## CI
 
 Several checks are configured to run on all commits using Github Actions, including lint, build and test steps. Definitions can be found in [./github/workflows](./github/workflows). Those checks which always run are required to be successful for PRs to be mergable.

Rakefile

@@ -1,4 +1,4 @@
-require 'rake'
+require "rake"
 
 desc "run application"
 task :run do
@@ -17,7 +17,7 @@ task :run do
   end
 end
 
-{ production: 'flaredown', staging: 'flaredown-staging'}.each do |env, application|
+{production: "flaredown", staging: "flaredown-staging"}.each do |env, application|
   namespace env.to_sym do
     desc "restart application"
     task :restart do
@@ -27,21 +27,21 @@ end
 
     desc "deploy application"
     task :deploy do
-      Rake::Task["#{env.to_s}:deploy:backend"].invoke
-      Rake::Task["#{env.to_s}:deploy:frontend"].invoke
+      Rake::Task["#{env}:deploy:backend"].invoke
+      Rake::Task["#{env}:deploy:frontend"].invoke
     end
 
     namespace :deploy do
       desc "deploy frontend application"
       task :frontend do
         log "Deploy frontend #{application} with revision: #{revision}"
-        deploy_to "[email protected]:#{application}-webapp.git",  'frontend'
+        deploy_to "[email protected]:#{application}-webapp.git", "frontend"
       end
 
       desc "deploy backend application"
       task :backend do
         log "Deploy backend #{application} with revision: #{revision}"
-        deploy_to "[email protected]:#{application}-api.git",  'backend'
+        deploy_to "[email protected]:#{application}-api.git", "backend"
         migrate "#{application}-api"
       end
     end
@@ -57,7 +57,6 @@ end
       system("heroku run rake app:invite --app #{application}-api")
     end
   end
-
 end
 
 def deploy_to(remote, subtree)
@@ -73,7 +72,7 @@ def restart(application)
 end
 
 def revision
-  ENV.fetch('REVISION') {'master'}
+  ENV.fetch("REVISION") { "master" }
 end
 
 def log(message)

backend/Dockerfile

@@ -14,7 +14,7 @@ RUN gem install bundler:2.1.4
 COPY Gemfile Gemfile.lock ./
 
 # install the gems
-RUN bundle install
+RUN bundle install --full-index
 
 # copy the rest of the application files to the container
 COPY . .

backend/Gemfile

@@ -48,8 +48,8 @@ gem "globalize"
 # Abort requests that are taking too long
 gem "rack-timeout"
 
-# wrapper for forecast.io API, Dark Sky
-gem "forecast_io"
+# wrapper for tomorrow.io API
+gem "tomorrowio_rb", "~>0.0.3"
 
 gem "geocoder"
 gem "nearest_time_zone"

backend/Gemfile.lock

@@ -170,10 +170,6 @@ GEM
     faraday-rack (1.0.0)
     ffaker (2.21.0)
     ffi (1.15.5-java)
-    forecast_io (2.0.2)
-      faraday
-      hashie
-      multi_json
     foreman (0.87.2)
     geocoder (1.7.0)
     globalid (1.2.1)
@@ -440,6 +436,7 @@ GEM
     thread_safe (0.3.6)
     thread_safe (0.3.6-java)
     timeout (0.4.0)
+    tomorrowio_rb (0.0.3)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     tzinfo-data (1.2021.3)
@@ -488,7 +485,6 @@ DEPENDENCIES
   erb_lint
   factory_bot_rails
   ffaker
-  forecast_io
   foreman
   geocoder
   globalize
@@ -519,6 +515,7 @@ DEPENDENCIES
   simplecov
   standardrb
   symmetric-encryption
+  tomorrowio_rb (~> 0.0.3)
   tzinfo-data
   vcr
   webmock

backend/app/models/search.rb

@@ -58,7 +58,6 @@ def where_conditions
             end
           conditions << ["lower(#{key}) LIKE ?", pattern]
         end
-        conditions
       end
   end
 

backend/app/services/charts_pattern.rb

@@ -76,7 +76,7 @@ def data(chart)
 
     average = collected_data.inject(0.0) { |result, datum| result + datum[:y].to_f } / collected_data.count
 
-    [{ x: collected_data[0]&.fetch(:x), y: average }]
+    [{x: collected_data[0]&.fetch(:x), y: average}]
   end
 
   def static_trackables_coordinates(category, checkin_ids, id)
@@ -88,7 +88,7 @@ def find_coordinates_by_checkin(category, checkin_ids, id)
     category_name = category.singularize
 
     "Checkin::#{category_name.camelize}".constantize.where(
-      checkin_id: {'$in': checkin_ids},
+      checkin_id: {"$in": checkin_ids},
       "#{category_name}_id": id
     ).map do |tr|
       coord = {x: tr.checkin.date, y: tr.value}

backend/app/services/weather_retriever.rb

@@ -14,47 +14,80 @@ def get(date, postal_code)
       end
 
       forecast = get_forecast(date, position)
-      the_day = forecast&.daily&.data&.first
 
-      if the_day.blank?
-        Rails.logger.warn "No forecast found for position #{position.inspect}: #{forecast.inspect}"
+      if forecast.status != 200
+        Rails.logger.warn "No forecast found for position #{position.inspect}: response code was #{forecast.status}, headers were #{forecast.headers}, body contained #{forecast.body}"
 
         return
       end
 
-      create_weather(the_day, position.id)
+      create_weather(forecast, position.id)
     end
 
     private
 
     def get_forecast(date, position)
-      tz = Time.zone
-      Time.zone = NearestTimeZone.to(position.latitude, position.longitude)
-
-      forecast = ForecastIO.forecast(
-        position.latitude,
-        position.longitude,
-        time: Time.zone.parse(date.to_s).to_i,
-        params: {exclude: "currently,minutely,hourly,alerts,flags"}
+      Tomorrowiorb.forecast(
+        "#{position.latitude},#{position.longitude}",
+        ["1d"],
+        "metric"
       )
-
-      Time.zone = tz
-
-      forecast
     end
 
-    def create_weather(the_day, position_id)
+    def create_weather(forecast, position_id)
+      today = JSON.parse(forecast.body, symbolize_names: true).dig(:timelines, :daily, 0)
+      the_time = today.dig(:time)
+      today = today.dig(:values)
+      rain_intensity = today.dig(:rainIntensityAvg)
+      sleet_intensity = today.dig(:sleetIntensityAvg)
+      snow_intensity = today.dig(:snowIntensityAvg)
+      icon = get_icon_legacy(today)
+      summary = "General conditions are #{icon}, with an average temperature of #{today[:temperatureAvg]}."
       Weather.create(
-        date: Date.strptime(the_day.time.to_s, "%s"),
-        humidity: (the_day.humidity * 100).round,
-        icon: the_day.icon,
+        date: Date.strptime(the_time, "%Y-%m-%d"),
+        humidity: today.dig(:humidityAvg).round,
+        icon: icon,
         position_id: position_id,
-        precip_intensity: the_day.precipIntensity,
-        pressure: the_day.pressure.round,
-        summary: the_day.summary,
-        temperature_max: the_day.temperatureMax.round,
-        temperature_min: the_day.temperatureMin.round
+        precip_intensity: rain_intensity + sleet_intensity + snow_intensity,
+        pressure: today.dig(:pressureSurfaceLevelAvg),
+        summary: summary,
+        temperature_max: today.dig(:temperatureMax),
+        temperature_min: today.dig(:temperatureMin)
       )
     end
+
+    def get_icon_legacy(today)
+      # Our icons do not coverage their full range of weather codes. We could pull in their icons (linked below) on the frontend to expand options
+      # This method adapts their weather codes to our existing icons as best as possible
+      # Icons and codes found here: https://docs.tomorrow.io/reference/data-layers-weather-codes
+      # Icon files here: https://github.com/Tomorrow-IO-API/tomorrow-weather-codes
+      # Daily forecast is always daytime weather codes / icons regardless of actual time
+      code = if today["weatherCodeMin"]
+        today["weatherCodeMin"]
+      elsif today["weatherCodeFullDay"]
+        today["weatherCodeFullDay"]
+      else
+        today["weatherCode"]
+      end
+
+      case code
+      when 1000, 1100
+        "clear-day"
+      when 1101
+        "partly-cloudy-day"
+      when 1102, 1001, 8000
+        "cloudy"
+      when 2000, 2100
+        "fog"
+      when 4000, 4001, 4200, 4201
+        "rain"
+      when 5000, 5001, 5100, 5101
+        "snow"
+      when 6000, 6001, 6200, 6201, 7000, 7101, 7102
+        "sleet"
+      else
+        "default"
+      end
+    end
   end
 end

backend/bin/bundle

@@ -1,3 +1,3 @@
 #!/usr/bin/env ruby
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
-load Gem.bin_path('bundler', 'bundle')
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __FILE__)
+load Gem.bin_path("bundler", "bundle")

backend/bin/localtunnel

@@ -6,20 +6,20 @@ def shut_down
 end
 
 # Trap ^C
-Signal.trap('INT') do
+Signal.trap("INT") do
   shut_down
   exit
 end
 
 # Trap `Kill `
-Signal.trap('TERM') do
+Signal.trap("TERM") do
   shut_down
   exit
 end
 
-if ENV['LOCALTUNNEL_ENABLED'] && ENV['LOCALTUNNEL_ENABLED'].eql?('true')
+if ENV["LOCALTUNNEL_ENABLED"] && ENV["LOCALTUNNEL_ENABLED"].eql?("true")
   begin
-    system "lt --port #{ENV['LOCALTUNNEL_PORT']} --subdomain #{ENV['LOCALTUNNEL_SUBDOMAIN']}"
+    system "lt --port #{ENV["LOCALTUNNEL_PORT"]} --subdomain #{ENV["LOCALTUNNEL_SUBDOMAIN"]}"
   rescue => e
     puts e.message
     retry

backend/bin/rails

@@ -1,9 +1,9 @@
 #!/usr/bin/env ruby
-if ENV['RAILS_ENV'] == 'test'
-  require 'simplecov'
-  SimpleCov.start 'rails'
+if ENV["RAILS_ENV"] == "test"
+  require "simplecov"
+  SimpleCov.start "rails"
   puts "required simplecov"
 end
-APP_PATH = File.expand_path('../../config/application', __FILE__)
-require_relative '../config/boot'
-require 'rails/commands'
+APP_PATH = File.expand_path("../../config/application", __FILE__)
+require_relative "../config/boot"
+require "rails/commands"

backend/bin/rake

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
-require_relative '../config/boot'
-require 'rake'
+require_relative "../config/boot"
+require "rake"
 Rake.application.run

backend/bin/rspec

@@ -6,11 +6,11 @@
 # this file is here to facilitate running it.
 #
 
-require 'pathname'
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile',
-                                           Pathname.new(__FILE__).realpath)
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
 
-require 'rubygems'
-require 'bundler/setup'
+require "rubygems"
+require "bundler/setup"
 
-load Gem.bin_path('rspec-core', 'rspec')
+load Gem.bin_path("rspec-core", "rspec")

backend/bin/setup

@@ -1,10 +1,10 @@
 #!/usr/bin/env ruby
-require 'pathname'
-require 'fileutils'
+require "pathname"
+require "fileutils"
 include FileUtils
 
 # path to your application root.
-APP_ROOT = Pathname.new File.expand_path('../../', __FILE__)
+APP_ROOT = Pathname.new File.expand_path("../../", __FILE__)
 
 def system!(*args)
   system(*args) || abort("\n== Command #{args} failed ==")
@@ -14,21 +14,21 @@ chdir APP_ROOT do
   # This script is a starting point to setup your application.
   # Add necessary setup steps to this file.
 
-  puts '== Installing dependencies =='
-  system! 'gem install bundler --conservative'
-  system('bundle check') or system!('bundle install')
+  puts "== Installing dependencies =="
+  system! "gem install bundler --conservative"
+  system("bundle check") or system!("bundle install")
 
   # puts "\n== Copying sample files =="
   # unless File.exist?('config/database.yml')
   #   cp 'config/database.yml.sample', 'config/database.yml'
   # end
 
   puts "\n== Preparing database =="
-  system! 'bin/rails db:setup'
+  system! "bin/rails db:setup"
 
   puts "\n== Removing old logs and tempfiles =="
-  system! 'bin/rails log:clear tmp:clear'
+  system! "bin/rails log:clear tmp:clear"
 
   puts "\n== Restarting application server =="
-  system! 'bin/rails restart'
+  system! "bin/rails restart"
 end