BXC-5771 - Improve missing mount detection

lib/pinecone/preservation_location.rb

@@ -1,60 +1,89 @@
 require 'yaml'
 require_relative 'environment'
+require_relative 'preservation_location_unavailable_error'
 
 module Pinecone
   class PreservationLocation
     attr_accessor :path
     attr_accessor :bag_pattern
     attr_accessor :base_path
     attr_reader :info, :loc_key
-    
+
     def initialize(name, info)
       @base_path = File.absolute_path info["base_path"]
       @bag_pattern = info["bag_pattern"]
       @path = File.join(@base_path, @bag_pattern)
       @loc_key = name
       @info = info
     end
-    
+
     def is_available()
-      return @base_path != nil && File.directory?(@base_path) && File.readable?(@base_path)
+      if @base_path == nil || !File.directory?(@base_path) || !File.readable?(@base_path)
+        return false
+      end
+
+      if !Dir.children(@base_path).empty?
+        return true
+      end
+
+      db = Pinecone::Environment.get_db
+      if db == nil
+        return false
+      end
+
+      begin
+        row = db.get_first_row(
+          "select 1 from bags where path like ? and (isReplica is null or isReplica = 0) limit 1",
+          [File.join(@base_path, "%")]
+        )
+        return row == nil
+      rescue SQLite3::SQLException
+        return false
+      end
     end
-
+
+    def assert_available()
+      if !is_available
+          raise PreservationLocationUnavailableError,
+            "Preservation location #{@loc_key} at #{@base_path} is unavailable"
+      end
+    end
+
     # Returns the list of email addresses to contact for this location
     def get_contact_emails
       if @info == nil || !(@info.key? "contacts")
         return Array.new
       end
-      
+
       return @info["contacts"]
     end
-    
+
     # Returns a list of bag paths within this location
     def get_bag_paths
       bag_paths = Array.new
-      
+
       Dir.glob(@path).each do |entry|
         if !(File.directory? entry) || entry == "." || entry == ".."
           next
         end
-  
+
         bag_paths.push entry
       end
-      
+
       return bag_paths
     end
-    
+
     # Returns the replica path for this location
     def get_replica_path(replica_base_path)
       return File.join(replica_base_path, @loc_key)
     end
-    
+
     # Gets the relative path for bag's path versus this location
     def get_relative_path(bag_path_string)
       bag_path = Pathname.new bag_path_string
       loc_path = Pathname.new @base_path
-      
+
       return bag_path.relative_path_from(loc_path)
     end
   end
-end
+end

lib/pinecone/preservation_location_manager.rb

@@ -8,13 +8,13 @@ class PreservationLocationManager
     attr_reader :pres_locs
     :replica_paths
     :logger
-    
+
     def initialize(loc_configs, replica_paths)
       @logger = Pinecone::Environment.logger
-      
+
       @replica_paths = replica_paths
       @pres_locs = Hash.new
-      
+
       loc_configs.each do |name, config|
         loc = Pinecone::PreservationLocation.new(name, config)
         @pres_locs[loc.base_path] = loc
@@ -23,32 +23,36 @@ def initialize(loc_configs, replica_paths)
         end
       end
     end
-    
+
     def get_bag_paths
       bag_paths = Array.new
 
       @pres_locs.each do |path, loc|
-        bag_paths = bag_paths + loc.get_bag_paths
+        loc.assert_available
+
+        bag_paths.concat(loc.get_bag_paths)
       end
-      
+
       return bag_paths
     end
-    
+
     def get_location_by_path(bag_path)
       @pres_locs.each do |loc_base, loc|
         if File.fnmatch loc.path, bag_path
+          loc.assert_available
           return loc
         end
       end
-      
+
       @replica_paths.each do |replica_path|
         @pres_locs.each do |loc_base, loc|
           if bag_path.start_with? loc.get_replica_path(replica_path)
+            loc.assert_available
             return loc
           end
         end
       end
-      
+
       return nil
     end
   end

lib/pinecone/preservation_location_unavailable_error.rb

@@ -0,0 +1,4 @@
+module Pinecone
+  class PreservationLocationUnavailableError < StandardError
+  end
+end

test/test_preservation_location.rb

@@ -1,12 +1,30 @@
 require "test/unit"
+require 'fileutils'
+require 'tmpdir'
+require_relative '../lib/pinecone/setup'
+require_relative '../lib/pinecone/environment'
 require_relative '../lib/pinecone/preservation_location'
 
 class TestPreservationLocation < Test::Unit::TestCase
   :loc_config
-
+   
   def setup
-    config = YAML.load_file("test-data/config.yaml")
-    @loc_config = config["preservation_locations"]
+    @tmp_test_dir = Dir.mktmpdir
+    FileUtils.cp("test-data/config.yaml", @tmp_test_dir)
+    @test_data = File.join(@tmp_test_dir, "test-data")
+    FileUtils.mkdir(@test_data)
+    FileUtils.cp_r("test-data/simple-loc", @test_data)
+    FileUtils.cp_r("test-data/invalid-loc", @test_data)
+
+    Pinecone::Environment.setup_env(@tmp_test_dir)
+    Pinecone::setup_database
+
+    @db = Pinecone::Environment.get_db
+    @loc_config = Pinecone::Environment.get_preservation_locations
+  end
+
+  def teardown
+    FileUtils.rm_r @tmp_test_dir
   end
 
   def test_get_contact_emails
@@ -50,13 +68,57 @@ def test_is_available
 
     assert_true(loc.is_available)
   end
+
+  def test_assert_available
+    loc = Pinecone::PreservationLocation.new("simple-tps-loc",
+        @loc_config["simple-tps-loc"])
+
+    assert_nothing_raised do
+      loc.assert_available
+    end
+  end
 
   def test_is_unavailable
-    loc_config = @loc_config["simple-tps-loc"]
+    loc_config = @loc_config["simple-tps-loc"].dup
     loc_config["base_path"] = loc_config["base_path"] + "_bad"
     loc = Pinecone::PreservationLocation.new("simple-tps-loc",
         loc_config)
 
     assert_false(loc.is_available)
   end
-end
+
+  def test_assert_available_unavailable
+    loc_config = @loc_config["simple-tps-loc"].dup
+    loc_config["base_path"] = loc_config["base_path"] + "_bad"
+    loc = Pinecone::PreservationLocation.new("simple-tps-loc",
+        loc_config)
+
+    error = assert_raise(Pinecone::PreservationLocationUnavailableError) do
+      loc.assert_available
+    end
+
+    assert_equal("Preservation location simple-tps-loc at #{loc.base_path} is unavailable", error.message)
+  end
+
+  def test_is_available_empty_location_with_no_db_contents
+    empty_path = File.join(@test_data, "empty-loc")
+    FileUtils.mkdir(empty_path)
+    loc_config = @loc_config["simple-tps-loc"].dup
+    loc_config["base_path"] = empty_path
+    loc = Pinecone::PreservationLocation.new("empty-loc", loc_config)
+
+    assert_true(loc.is_available)
+  end
+
+  def test_is_unavailable_empty_location_with_db_contents
+    empty_path = File.join(@test_data, "empty-loc")
+    FileUtils.mkdir(empty_path)
+    @db.execute("insert into bags (path, valid, lastValidated, isReplica) values (?, 1, CURRENT_TIMESTAMP, 0)",
+        [File.join(empty_path, "missing_bag")])
+    loc_config = @loc_config["simple-tps-loc"].dup
+    loc_config["base_path"] = empty_path
+    loc = Pinecone::PreservationLocation.new("empty-loc", loc_config)
+
+    assert_false(loc.is_available)
+  end
+end

test/test_preservation_location_manager.rb

@@ -1,4 +1,8 @@
 require "test/unit"
+require 'fileutils'
+require 'tmpdir'
+require_relative '../lib/pinecone/setup'
+require_relative '../lib/pinecone/environment'
 require_relative '../lib/pinecone/preservation_location_manager'
 
 class TestPreservationLocationManager < Test::Unit::TestCase
@@ -18,6 +22,7 @@ def setup
     FileUtils.cp_r("test-data/invalid-loc", @@invalid_abs)
 
     Pinecone::Environment.setup_env(@tmp_test_dir)
+    Pinecone::setup_database
 
     #config = YAML.load_file(File.join(@tmp_test_dir, "config.yaml"))
     #@loc_config = config["preservation_locations"]
@@ -45,6 +50,17 @@ def test_get_location_by_path
     assert_not_nil(loc)
     assert_equal(@@invalid_abs, loc.base_path)
   end
+
+  def test_get_location_by_path_unavailable_location
+    manager = Pinecone::PreservationLocationManager.new @loc_config, []
+    FileUtils.rm_rf @@invalid_abs
+
+    error = assert_raise(Pinecone::PreservationLocationUnavailableError) do
+      manager.get_location_by_path File.join(@@invalid_abs, "incomplete_bag")
+    end
+
+    assert_equal("Preservation location invalid-loc at #{@@invalid_abs} is unavailable", error.message)
+  end
 
   def test_get_location_by_path_invalid_location
     @loc_config.delete("invalid-loc")
@@ -77,6 +93,17 @@ def test_get_bag_paths_multiple_locations
     bag_paths = manager.get_bag_paths
     assert_equal(4, bag_paths.length)
   end
+
+  def test_get_bag_paths_unavailable_location
+    manager = Pinecone::PreservationLocationManager.new(@loc_config, ["./replicas"])
+    FileUtils.rm_rf @@simple_abs
+
+    error = assert_raise(Pinecone::PreservationLocationUnavailableError) do
+      manager.get_bag_paths
+    end
+
+    assert_equal("Preservation location simple-tps-loc at #{@@simple_abs} is unavailable", error.message)
+  end
 
   def test_unreachable_location
     simple_loc = File.join(@test_data, "simple-loc")
@@ -86,4 +113,19 @@ def test_unreachable_location
       manager = Pinecone::PreservationLocationManager.new(@loc_config, ["./replicas"])
     end
   end
-end
+
+  def test_empty_location_with_db_contents_unavailable
+    empty_loc = File.join(@test_data, "empty-loc")
+    FileUtils.mkdir(empty_loc)
+    @db = Pinecone::Environment.get_db
+    @db.execute("insert into bags (path, valid, lastValidated, isReplica) values (?, 1, CURRENT_TIMESTAMP, 0)",
+        [File.join(empty_loc, "missing_bag")])
+
+    loc_config = @loc_config.transform_values(&:dup)
+    loc_config["simple-tps-loc"]["base_path"] = empty_loc
+
+    assert_raise ArgumentError do
+      manager = Pinecone::PreservationLocationManager.new(loc_config, ["./replicas"])
+    end
+  end
+end