diff --git a/lib/pinecone/preservation_location.rb b/lib/pinecone/preservation_location.rb index 9881e43..fdf30aa 100644 --- a/lib/pinecone/preservation_location.rb +++ b/lib/pinecone/preservation_location.rb @@ -1,5 +1,6 @@ require 'yaml' require_relative 'environment' +require_relative 'preservation_location_unavailable_error' module Pinecone class PreservationLocation @@ -7,7 +8,7 @@ class PreservationLocation attr_accessor :bag_pattern attr_accessor :base_path attr_reader :info, :loc_key - + def initialize(name, info) @base_path = File.absolute_path info["base_path"] @bag_pattern = info["bag_pattern"] @@ -15,46 +16,74 @@ def initialize(name, info) @loc_key = name @info = info end - + def is_available() - return @base_path != nil && File.directory?(@base_path) && File.readable?(@base_path) + if @base_path == nil || !File.directory?(@base_path) || !File.readable?(@base_path) + return false + end + + if !Dir.children(@base_path).empty? + return true + end + + db = Pinecone::Environment.get_db + if db == nil + return false + end + + begin + row = db.get_first_row( + "select 1 from bags where path like ? and (isReplica is null or isReplica = 0) limit 1", + [File.join(@base_path, "%")] + ) + return row == nil + rescue SQLite3::SQLException + return false + end end - + + def assert_available() + if !is_available + raise PreservationLocationUnavailableError, + "Preservation location #{@loc_key} at #{@base_path} is unavailable" + end + end + # Returns the list of email addresses to contact for this location def get_contact_emails if @info == nil || !(@info.key? "contacts") return Array.new end - + return @info["contacts"] end - + # Returns a list of bag paths within this location def get_bag_paths bag_paths = Array.new - + Dir.glob(@path).each do |entry| if !(File.directory? entry) || entry == "." || entry == ".." next end - + bag_paths.push entry end - + return bag_paths end - + # Returns the replica path for this location def get_replica_path(replica_base_path) return File.join(replica_base_path, @loc_key) end - + # Gets the relative path for bag's path versus this location def get_relative_path(bag_path_string) bag_path = Pathname.new bag_path_string loc_path = Pathname.new @base_path - + return bag_path.relative_path_from(loc_path) end end -end \ No newline at end of file +end diff --git a/lib/pinecone/preservation_location_manager.rb b/lib/pinecone/preservation_location_manager.rb index ed15fb8..8e3b399 100644 --- a/lib/pinecone/preservation_location_manager.rb +++ b/lib/pinecone/preservation_location_manager.rb @@ -8,13 +8,13 @@ class PreservationLocationManager attr_reader :pres_locs :replica_paths :logger - + def initialize(loc_configs, replica_paths) @logger = Pinecone::Environment.logger - + @replica_paths = replica_paths @pres_locs = Hash.new - + loc_configs.each do |name, config| loc = Pinecone::PreservationLocation.new(name, config) @pres_locs[loc.base_path] = loc @@ -23,32 +23,36 @@ def initialize(loc_configs, replica_paths) end end end - + def get_bag_paths bag_paths = Array.new @pres_locs.each do |path, loc| - bag_paths = bag_paths + loc.get_bag_paths + loc.assert_available + + bag_paths.concat(loc.get_bag_paths) end - + return bag_paths end - + def get_location_by_path(bag_path) @pres_locs.each do |loc_base, loc| if File.fnmatch loc.path, bag_path + loc.assert_available return loc end end - + @replica_paths.each do |replica_path| @pres_locs.each do |loc_base, loc| if bag_path.start_with? loc.get_replica_path(replica_path) + loc.assert_available return loc end end end - + return nil end end diff --git a/lib/pinecone/preservation_location_unavailable_error.rb b/lib/pinecone/preservation_location_unavailable_error.rb new file mode 100644 index 0000000..72c2b93 --- /dev/null +++ b/lib/pinecone/preservation_location_unavailable_error.rb @@ -0,0 +1,4 @@ +module Pinecone + class PreservationLocationUnavailableError < StandardError + end +end \ No newline at end of file diff --git a/test/test_preservation_location.rb b/test/test_preservation_location.rb index ba9a40b..e9102b0 100644 --- a/test/test_preservation_location.rb +++ b/test/test_preservation_location.rb @@ -1,12 +1,30 @@ require "test/unit" +require 'fileutils' +require 'tmpdir' +require_relative '../lib/pinecone/setup' +require_relative '../lib/pinecone/environment' require_relative '../lib/pinecone/preservation_location' class TestPreservationLocation < Test::Unit::TestCase :loc_config - + def setup - config = YAML.load_file("test-data/config.yaml") - @loc_config = config["preservation_locations"] + @tmp_test_dir = Dir.mktmpdir + FileUtils.cp("test-data/config.yaml", @tmp_test_dir) + @test_data = File.join(@tmp_test_dir, "test-data") + FileUtils.mkdir(@test_data) + FileUtils.cp_r("test-data/simple-loc", @test_data) + FileUtils.cp_r("test-data/invalid-loc", @test_data) + + Pinecone::Environment.setup_env(@tmp_test_dir) + Pinecone::setup_database + + @db = Pinecone::Environment.get_db + @loc_config = Pinecone::Environment.get_preservation_locations + end + + def teardown + FileUtils.rm_r @tmp_test_dir end def test_get_contact_emails @@ -50,13 +68,57 @@ def test_is_available assert_true(loc.is_available) end + + def test_assert_available + loc = Pinecone::PreservationLocation.new("simple-tps-loc", + @loc_config["simple-tps-loc"]) + + assert_nothing_raised do + loc.assert_available + end + end def test_is_unavailable - loc_config = @loc_config["simple-tps-loc"] + loc_config = @loc_config["simple-tps-loc"].dup loc_config["base_path"] = loc_config["base_path"] + "_bad" loc = Pinecone::PreservationLocation.new("simple-tps-loc", loc_config) assert_false(loc.is_available) end -end \ No newline at end of file + + def test_assert_available_unavailable + loc_config = @loc_config["simple-tps-loc"].dup + loc_config["base_path"] = loc_config["base_path"] + "_bad" + loc = Pinecone::PreservationLocation.new("simple-tps-loc", + loc_config) + + error = assert_raise(Pinecone::PreservationLocationUnavailableError) do + loc.assert_available + end + + assert_equal("Preservation location simple-tps-loc at #{loc.base_path} is unavailable", error.message) + end + + def test_is_available_empty_location_with_no_db_contents + empty_path = File.join(@test_data, "empty-loc") + FileUtils.mkdir(empty_path) + loc_config = @loc_config["simple-tps-loc"].dup + loc_config["base_path"] = empty_path + loc = Pinecone::PreservationLocation.new("empty-loc", loc_config) + + assert_true(loc.is_available) + end + + def test_is_unavailable_empty_location_with_db_contents + empty_path = File.join(@test_data, "empty-loc") + FileUtils.mkdir(empty_path) + @db.execute("insert into bags (path, valid, lastValidated, isReplica) values (?, 1, CURRENT_TIMESTAMP, 0)", + [File.join(empty_path, "missing_bag")]) + loc_config = @loc_config["simple-tps-loc"].dup + loc_config["base_path"] = empty_path + loc = Pinecone::PreservationLocation.new("empty-loc", loc_config) + + assert_false(loc.is_available) + end +end diff --git a/test/test_preservation_location_manager.rb b/test/test_preservation_location_manager.rb index db7e5cf..ef7e89d 100644 --- a/test/test_preservation_location_manager.rb +++ b/test/test_preservation_location_manager.rb @@ -1,4 +1,8 @@ require "test/unit" +require 'fileutils' +require 'tmpdir' +require_relative '../lib/pinecone/setup' +require_relative '../lib/pinecone/environment' require_relative '../lib/pinecone/preservation_location_manager' class TestPreservationLocationManager < Test::Unit::TestCase @@ -18,6 +22,7 @@ def setup FileUtils.cp_r("test-data/invalid-loc", @@invalid_abs) Pinecone::Environment.setup_env(@tmp_test_dir) + Pinecone::setup_database #config = YAML.load_file(File.join(@tmp_test_dir, "config.yaml")) #@loc_config = config["preservation_locations"] @@ -45,6 +50,17 @@ def test_get_location_by_path assert_not_nil(loc) assert_equal(@@invalid_abs, loc.base_path) end + + def test_get_location_by_path_unavailable_location + manager = Pinecone::PreservationLocationManager.new @loc_config, [] + FileUtils.rm_rf @@invalid_abs + + error = assert_raise(Pinecone::PreservationLocationUnavailableError) do + manager.get_location_by_path File.join(@@invalid_abs, "incomplete_bag") + end + + assert_equal("Preservation location invalid-loc at #{@@invalid_abs} is unavailable", error.message) + end def test_get_location_by_path_invalid_location @loc_config.delete("invalid-loc") @@ -77,6 +93,17 @@ def test_get_bag_paths_multiple_locations bag_paths = manager.get_bag_paths assert_equal(4, bag_paths.length) end + + def test_get_bag_paths_unavailable_location + manager = Pinecone::PreservationLocationManager.new(@loc_config, ["./replicas"]) + FileUtils.rm_rf @@simple_abs + + error = assert_raise(Pinecone::PreservationLocationUnavailableError) do + manager.get_bag_paths + end + + assert_equal("Preservation location simple-tps-loc at #{@@simple_abs} is unavailable", error.message) + end def test_unreachable_location simple_loc = File.join(@test_data, "simple-loc") @@ -86,4 +113,19 @@ def test_unreachable_location manager = Pinecone::PreservationLocationManager.new(@loc_config, ["./replicas"]) end end -end \ No newline at end of file + + def test_empty_location_with_db_contents_unavailable + empty_loc = File.join(@test_data, "empty-loc") + FileUtils.mkdir(empty_loc) + @db = Pinecone::Environment.get_db + @db.execute("insert into bags (path, valid, lastValidated, isReplica) values (?, 1, CURRENT_TIMESTAMP, 0)", + [File.join(empty_loc, "missing_bag")]) + + loc_config = @loc_config.transform_values(&:dup) + loc_config["simple-tps-loc"]["base_path"] = empty_loc + + assert_raise ArgumentError do + manager = Pinecone::PreservationLocationManager.new(loc_config, ["./replicas"]) + end + end +end