diff --git a/.circleci/config.yml b/.circleci/config.yml index b3140676..af456f3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,6 +9,10 @@ jobs: - image: perldocker/perl-tester:<< parameters.perl-version >> steps: - checkout + - run: + name: Install Redis server + command: | + apt-get install -y redis - run: command: cpm install -g --no-test Dist::Zilla Dist::Zilla::App::Command::cover ExtUtils::MakeMaker diff --git a/Changes b/Changes index 5ec4bc43..a9861210 100644 --- a/Changes +++ b/Changes @@ -1,6 +1,7 @@ Revision history for Data-Validate-Sanctions {{$NEXT}} + 0.13 2022-07-26 13:55:00 CST Improving the search for larger sanction lists diff --git a/cpanfile b/cpanfile index f788ee9c..59ca0bd2 100644 --- a/cpanfile +++ b/cpanfile @@ -19,6 +19,8 @@ requires 'Getopt::Long', '2.42'; requires 'Syntax::Keyword::Try', '0.18'; requires 'Locale::Country', '3.66'; requires 'Text::Trim', 0; +requires 'JSON::MaybeUTF8', 0; +requires 'Clone', 0; on test => sub { requires 'Test::More', '0.96'; @@ -26,7 +28,11 @@ on test => sub { requires 'Test::Warnings', '0.026'; requires 'Test::MockModule', '0.15'; requires 'Test::MockObject', '1.20161202'; + requires 'Test::MockTime'; + requires 'Test::Deep', '0'; requires 'FindBin', '0'; requires 'Path::Tiny', '0'; requires 'Class::Unload', '0'; + requires 'Test::RedisServer', '0.23'; + requires 'RedisDB', '2.57'; }; diff --git a/lib/Data/Validate/Sanctions.pm b/lib/Data/Validate/Sanctions.pm index 1bed060e..95e2e384 100644 --- a/lib/Data/Validate/Sanctions.pm +++ b/lib/Data/Validate/Sanctions.pm @@ -9,26 +9,33 @@ our @EXPORT_OK = qw/is_sanctioned set_sanction_file get_sanction_file/; use Carp; use Data::Validate::Sanctions::Fetcher; +use Data::Validate::Sanctions::Redis; use File::stat; use File::ShareDir; -use YAML::XS qw/DumpFile LoadFile/; +use YAML::XS qw/DumpFile LoadFile/; use Scalar::Util qw(blessed); use Date::Utility; use Data::Compare; use List::Util qw(any uniq max min); use Locale::Country; use Text::Trim qw(trim); +use Clone qw(clone); -our $VERSION = '0.13'; +our $VERSION = '0.14'; -my $sanction_file = _default_sanction_file(); +my $sanction_file; my $instance; # for OO sub new { ## no critic (RequireArgUnpacking) my ($class, %args) = @_; + my $storage = delete $args{storage} // ''; + + return Data::Validate::Sanctions::Redis->new(%args) if $storage eq 'redis'; + my $self = {}; + $self->{sanction_file} = $args{sanction_file} // _default_sanction_file(); $self->{args} = {%args}; @@ -43,18 +50,27 @@ sub update_data { $self->_load_data(); my $new_data = Data::Validate::Sanctions::Fetcher::run($self->{args}->%*, %args); - - my $updated; + my $updated = 0; foreach my $k (keys %$new_data) { $self->{_data}->{$k} //= {}; $self->{_data}->{$k}->{updated} //= 0; $self->{_data}->{$k}->{content} //= []; - if ($self->{_data}{$k}->{updated} != $new_data->{$k}->{updated} + + if (!$new_data->{$k}->{error} && $self->{_data}->{$k}->{error}) { + delete $self->{_data}->{$k}->{error}; + $updated = 1; + } + + if ($new_data->{$k}->{error}) { + warn "$k list update failed because: $new_data->{$k}->{error}"; + $self->{_data}->{$k}->{error} = $new_data->{$k}->{error}; + $updated = 1; + } elsif ($self->{_data}{$k}->{updated} != $new_data->{$k}->{updated} || scalar $self->{_data}{$k}->{content}->@* != scalar $new_data->{$k}->{content}->@*) { + print "Source $k is updated with new data \n" if $args{verbose}; $self->{_data}->{$k} = $new_data->{$k}; $updated = 1; - print "Source $k is updated with new data \n" if $args{verbose}; } else { print "Source $k is not changed \n" if $args{verbose}; } @@ -76,7 +92,7 @@ sub last_updated { return $self->{_data}->{$list}->{updated}; } else { $self->_load_data(); - return max(map { $_->{updated} } values %{$self->{_data}}); + return max(map { $_->{updated} // 0 } values %{$self->{_data}}); } } @@ -87,6 +103,7 @@ sub set_sanction_file { ## no critic (RequireArgUnpacking) } sub get_sanction_file { + $sanction_file //= _default_sanction_file(); return $instance ? $instance->{sanction_file} : $sanction_file; } @@ -103,6 +120,14 @@ sub is_sanctioned { ## no critic (RequireArgUnpacking) return (get_sanctioned_info(@_))->{matched}; } +sub data { + my ($self) = @_; + + $self->_load_data() unless $self->{_data}; + + return $self->{_data}; +} + =head2 _match_other_fields Matches fields possibly available in addition to name and date of birth. @@ -173,9 +198,12 @@ It returns a hash-ref containg the following data: =over 4 =item - matched: 1 if a match was found; 0 otherwise - list: the source for the matched entry, - matched_args: a name-value hash-ref of the similar arguments, - comment: additional comments if necessary, + +=item - list: the source for the matched entry, + +=item - matched_args: a name-value hash-ref of the similar arguments, + +=item - comment: additional comments if necessary, =back @@ -184,7 +212,7 @@ It returns a hash-ref containg the following data: sub get_sanctioned_info { ## no critic (RequireArgUnpacking) my $self = blessed($_[0]) ? shift : $instance; unless ($self) { - $instance = __PACKAGE__->new(sanction_file => $sanction_file); + $instance = __PACKAGE__->new(sanction_file => get_sanction_file()); $self = $instance; } @@ -219,7 +247,7 @@ sub get_sanctioned_info { ## no critic (RequireArgUnpacking) # and deduplicate the list my $filtered_sanctioned_names = {}; foreach my $token (@client_name_tokens) { - foreach my $name ( keys %{$self->{_token_sanctioned_names}->{$token}}) { + foreach my $name (keys %{$self->{_token_sanctioned_names}->{$token}}) { $filtered_sanctioned_names->{$name} = 1; } } @@ -286,12 +314,12 @@ sub get_sanctioned_info { ## no critic (RequireArgUnpacking) } sub _load_data { - my $self = shift; - my $sanction_file = $self->{sanction_file}; - $self->{last_time} //= 0; - $self->{_data} //= {}; - $self->{_sanctioned_name_tokens} //= {}; - $self->{_token_sanctioned_names} //= {}; + my $self = shift; + my $sanction_file = $self->{sanction_file}; + $self->{last_time} //= 0; + $self->{_data} //= {}; + $self->{_sanctioned_name_tokens} //= {}; + $self->{_token_sanctioned_names} //= {}; if (-e $sanction_file) { return $self->{_data} if stat($sanction_file)->mtime <= $self->{last_time} && $self->{_data}; @@ -303,8 +331,8 @@ sub _load_data { foreach my $sanctioned_name (keys $self->{_index}->%*) { my @tokens = _clean_names($sanctioned_name); $self->{_sanctioned_name_tokens}->{$sanctioned_name} = \@tokens; - foreach my $token (@tokens){ - $self->{_token_sanctioned_names}->{$token}->{$sanctioned_name}=1; + foreach my $token (@tokens) { + $self->{_token_sanctioned_names}->{$token}->{$sanctioned_name} = 1; } } @@ -320,10 +348,11 @@ Indexes data by name. Each name may have multiple matching entries. sub _index_data { my $self = shift; + $self->{_data} //= {}; $self->{_index} = {}; for my $source (keys $self->{_data}->%*) { - my @content = ($self->{_data}->{$source}->{content} // [])->@*; - warn "Content is empty for the sanction source $source. The sanctions file should be updated." unless @content; + my @content = clone($self->{_data}->{$source}->{content} // [])->@*; + for my $entry (@content) { $entry->{source} = $source; for my $name ($entry->{names}->@*) { @@ -392,6 +421,12 @@ sub _name_matches { return 0; } +sub export_data { + my ($self, $path) = @_; + + return DumpFile($path, $self->{_data}); +} + 1; __END__ @@ -487,6 +522,15 @@ set sanction_file which is used by L (procedure-oriented) Pass in the client's name and sanctioned individual's name to see if they are similar or not + +=head2 export_data + +Exports the sanction lists to a local file in YAML format. + +=head2 data + +Gets the sanction list content with lazy loading. + =head1 AUTHOR Binary.com Efayland@binary.comE diff --git a/lib/Data/Validate/Sanctions/Fetcher.pm b/lib/Data/Validate/Sanctions/Fetcher.pm index d83d8c8d..3e8a9165 100644 --- a/lib/Data/Validate/Sanctions/Fetcher.pm +++ b/lib/Data/Validate/Sanctions/Fetcher.pm @@ -6,7 +6,7 @@ use warnings; use DateTime::Format::Strptime; use Date::Utility; use IO::Uncompress::Unzip qw(unzip $UnzipError); -use List::Util qw(uniq any); +use List::Util qw(uniq any); use Mojo::UserAgent; use Text::CSV; use Text::Trim qw(trim); @@ -14,7 +14,7 @@ use Syntax::Keyword::Try; use XML::Fast; use Locale::Country; -our $VERSION = '0.10'; +# VERSION =head2 config @@ -234,7 +234,7 @@ sub _ofac_xml { $ref->{publshInformation}{Publish_Date} =~ m/(\d{1,2})\/(\d{1,2})\/(\d{4})/ ? _date_to_epoch("$3-$1-$2") : undef; # publshInformation is a typo in ofac xml tags - die 'Publication date is invalid' unless defined $publish_epoch; + die "Corrupt data. Release date is invalid\n" unless defined $publish_epoch; my $parse_list_node = sub { my ($entry, $parent, $child, $attribute) = @_; @@ -301,16 +301,16 @@ sub _hmt_csv { my $raw_data = shift; my $dataset = []; - my $csv = Text::CSV->new({binary => 1}) or die "Cannot use CSV: " . Text::CSV->error_diag(); + my $csv = Text::CSV->new({binary => 1}) or die "Cannot use CSV: " . Text::CSV->error_diag() . "\n"; my @lines = split("\n", $raw_data); my $parsed = $csv->parse(trim(shift @lines)); my @info = $parsed ? $csv->fields() : (); - die 'Publication date was not found' unless @info && _date_to_epoch($info[1]); + die "Currupt data. Release date was not found\n" unless @info && _date_to_epoch($info[1]); my $publish_epoch = _date_to_epoch($info[1]); - die 'Publication date is invalid' unless defined $publish_epoch; + die "Currupt data. Release date is invalid\n" unless defined $publish_epoch; $parsed = $csv->parse(trim(shift @lines)); my @row = $csv->fields(); @@ -342,7 +342,7 @@ sub _hmt_csv { # Fields to be added in the new file format (https://redmine.deriv.cloud/issues/51922) # We can read these fields normally after the data is released in the new format my ($passport_no, $non_latin_alias); - $passport_no = $row[$column{'Passport Number'}] if defined $column{'Passport Number'}; + $passport_no = $row[$column{'Passport Number'}] if defined $column{'Passport Number'}; $non_latin_alias = $row[$column{'Name Non-Latin Script'}] if defined $column{'Name Non-Latin Script'}; _process_sanction_entry( @@ -393,7 +393,7 @@ sub _eu_xml { my @place_of_birth = map { $_->{'-countryIso2Code'} || () } $entry->{birthdate}->@*; my @citizen = map { $_->{'-countryIso2Code'} || () } $entry->{citizenship}->@*; my @residence = map { $_->{'-countryIso2Code'} || () } $entry->{address}->@*; - my @postal_code = map { $_->{'-zipCode'} || $_->{'-poBox'} || () } $entry->{address}->@*; + my @postal_code = map { $_->{'-zipCode'} || $_->{'-poBox'} || () } $entry->{address}->@*; my @nationality = map { $_->{'-countryIso2Code'} || () } $entry->{identification}->@*; my @national_id = map { $_->{'-identificationTypeCode'} eq 'id' ? $_->{'-number'} || () : () } $entry->{identification}->@*; my @passport_no = map { $_->{'-identificationTypeCode'} eq 'passport' ? $_->{'-number'} || () : () } $entry->{identification}->@*; @@ -415,7 +415,7 @@ sub _eu_xml { my @date_parts = split('T', $ref->{'-generationDate'} // ''); my $publish_epoch = _date_to_epoch($date_parts[0] // ''); - die 'Publication date is invalid' unless $publish_epoch; + die "Corrupt data. Release date is invalid\n" unless $publish_epoch; return { updated => $publish_epoch, @@ -440,7 +440,7 @@ sub run { foreach my $id (sort keys %$config) { my $source = $config->{$id}; try { - die "Url is empty for $id" unless $source->{url}; + die "Url is empty for $id\n" unless $source->{url}; my $raw_data; @@ -461,8 +461,8 @@ sub run { my $count = $data->{content}->@*; print "Source $id: $count entries fetched \n" if $args{verbose}; } - } catch { - warn "$id list update failed because: $@"; + } catch ($e) { + $result->{$id}->{error} = $e; } } @@ -480,7 +480,7 @@ sub _entries_from_file { my $entries; - open my $fh, '<', "$1" or die "Can't open $id file $1 $!"; + open my $fh, '<', "$1" or die "Can't open $id file $1 $!\n"; $entries = do { local $/; <$fh> }; close $fh; @@ -513,16 +513,16 @@ sub _entries_from_remote_src { try { my $resp = $ua->get($src_url); - die "File not downloaded for $id" if $resp->result->is_error; + die "File not downloaded for $id\n" if $resp->result->is_error; $entries = $resp->result->body; last; - } catch { - $error_log = $@; + } catch ($e) { + $error_log = $e; } } - return $entries // die "An error occurred while fetching data from '$src_url' due to $error_log"; + return $entries // die "An error occurred while fetching data from '$src_url' due to $error_log\n"; } 1; diff --git a/lib/Data/Validate/Sanctions/Redis.pm b/lib/Data/Validate/Sanctions/Redis.pm new file mode 100644 index 00000000..bbc6d20b --- /dev/null +++ b/lib/Data/Validate/Sanctions/Redis.pm @@ -0,0 +1,254 @@ +package Data::Validate::Sanctions::Redis; + +use strict; +use warnings; + +use parent 'Data::Validate::Sanctions'; + +use Data::Validate::Sanctions::Fetcher; +use Scalar::Util qw(blessed); +use List::Util qw(max); +use JSON::MaybeUTF8 qw(encode_json_utf8 decode_json_utf8); +use YAML::XS qw(DumpFile); +use Syntax::Keyword::Try; + +# VERSION + +sub new { + my ($class, %args) = @_; + + my $self = {}; + + $self->{connection} = $args{connection} or die 'Redis connection is missing'; + + $self->{sources} = [keys Data::Validate::Sanctions::Fetcher::config(eu_token => 'dummy')->%*]; + + $self->{args} = {%args}; + $self->{last_time} = 0; + my $object = bless $self, ref($class) || $class; + $object->_load_data(); + + return $object; +} + +sub set_sanction_file { + die 'Not applicable'; +} + +sub get_sanction_file { + die 'Not applicable'; +} + +sub get_sanctioned_info { + my $self = shift; + + die "This function can only be called on an object" unless $self; + + return Data::Validate::Sanctions::get_sanctioned_info($self, @_); +} + +sub _load_data { + my $self = shift; + + $self->{last_time} //= 0; + $self->{_data} //= {}; + $self->{_sanctioned_name_tokens} //= {}; + $self->{_token_sanctioned_names} //= {}; + + my $last_time = $self->{last_time}; + for my $source ($self->{sources}->@*) { + try { + $self->{_data}->{$source} //= {}; + + my ($content, $verified, $updated, $error) = $self->{connection}->hmget("SANCTIONS::$source", qw/content verified updated error/)->@*; + $updated //= 0; + my $current_update_date = $self->{_data}->{$source}->{updated} // 0; + next if $current_update_date && $updated <= $current_update_date; + + $self->{_data}->{$source}->{content} = decode_json_utf8($content // '[]'); + $self->{_data}->{$source}->{verified} = $verified // 0; + $self->{_data}->{$source}->{updated} = $updated; + $self->{_data}->{$source}->{error} = $error // ''; + $last_time = $updated if $updated > $last_time; + } catch ($e) { + $self->{_data}->{$source}->{content} = []; + $self->{_data}->{$source}->{updated} = 0; + $self->{_data}->{$source}->{verified} = 0; + $self->{_data}->{$source}->{error} = "Failed to load from Redis: $e"; + } + } + $self->{last_time} = $last_time; + + $self->_index_data(); + + foreach my $sanctioned_name (keys $self->{_index}->%*) { + my @tokens = Data::Validate::Sanctions::_clean_names($sanctioned_name); + $self->{_sanctioned_name_tokens}->{$sanctioned_name} = \@tokens; + foreach my $token (@tokens) { + $self->{_token_sanctioned_names}->{$token}->{$sanctioned_name} = 1; + } + } + + return $self->{_data}; +} + +sub _save_data { + my $self = shift; + + for my $source ($self->{sources}->@*) { + $self->{_data}->{$source}->{verified} = time; + $self->{connection}->hmset( + "SANCTIONS::$source", + updated => $self->{_data}->{$source}->{updated} // 0, + content => encode_json_utf8($self->{_data}->{$source}->{content} // []), + verified => $self->{_data}->{$source}->{verified}, + error => $self->{_data}->{$source}->{error} // '' + ); + } + + return; +} + +sub _default_sanction_file { + die 'Not applicable'; +} + +1; +__END__ + +=encoding utf-8 + +=head1 NAME + +Data::Validate::Sanctions::Redis - An extension of L that stores sanction data in redis. + +=head1 SYNOPSIS + ## no critic + use Data::Validate::Sanctions::Redis; + + my $validator = Data::Validate::Sanctions::Redis->new(connection => $redis_read); + + # to validate clients by their name + print 'BAD' if $validator->is_sanctioned("$last_name $first_name"); + # or by more profile data + print 'BAD' if $validator->get_sanctioned_info(first_name => $first_name, last_name => $last_name, date_of_birth => $date_of_birth)->{matched}; + + # to update the sanction dataset (needs redis write access) + my $validator = Data::Validate::Sanctions::Redis->new(connection => $redis_write); ## no critic + $validator->update_data(eu_token => $token); + + # create object from the parent (factory) class + my $validator = Data::Validate::Sanctions->new(storage => 'redis', connection => $redis_write); + +=head1 DESCRIPTION + +Data::Validate::Sanctions::Redis is a simple validitor to validate a name against sanctions lists. +For more details about the sanction sources please refer to the parent module L. + +=head1 METHODS + +=head2 new + +Create the object with the redis object: + + my $validator = Data::Validate::Sanctions::Redis->new(connection => $redis); + +=head2 is_sanctioned + +Checks if the input profile info matches a sanctioned entity. +The arguments are the same as those of B. + +It returns 1 if a match is found, otherwise 0. + +=cut + +=head2 get_sanctioned_info + +Tries to find a match a sanction entry matching the input profile args. +It takes arguments in two forms. In the new API, it takes a hashref containing the following named arguments: + +=over 4 + +=item * first_name: first name + +=item * last_name: last name + +=item * date_of_birth: (optional) date of birth as a string or epoch + +=item * place_of_birth: (optional) place of birth as a country name or code + +=item * residence: (optional) name or code of the country of residence + +=item * nationality: (optional) name or code of the country of nationality + +=item * citizen: (optional) name or code of the country of citizenship + +=item * postal_code: (optional) postal/zip code + +=item * national_id: (optional) national ID number + +=item * passport_no: (oiptonal) passort number + +=back + +For backward compatibility it also supports the old API, taking the following args: + +=over 4 + +=item * first_name: first name + +=item * last_name: last name + +=item * date_of_birth: (optional) date of birth as a string or epoch + +=back + +It returns a hash-ref containg the following data: + +=over 4 + +=item - matched: 1 if a match was found; 0 otherwise + +=item - list: the source for the matched entry, + +=item - matched_args: a name-value hash-ref of the similar arguments, + +=item - comment: additional comments if necessary, + +=back + +=cut + +=head2 update_data + +Fetches latest versions of sanction lists, and updates corresponding sections of stored file, if needed + +=head2 last_updated + +Returns timestamp of when the latest list was updated. +If argument is provided - return timestamp of when that list was updated. + +=head2 _name_matches + +Pass in the client's name and sanctioned individual's name to see if they are similar or not + +=head1 AUTHOR + +Binary.com Efayland@binary.comE + +=head1 COPYRIGHT + +Copyright 2022- Binary.com + +=head1 LICENSE + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=head1 SEE ALSO + +L + +L + +=cut diff --git a/t/05_basic.t b/t/01_basic.t similarity index 71% rename from t/05_basic.t rename to t/01_basic.t index 6b9f6b89..87f22ef5 100644 --- a/t/05_basic.t +++ b/t/01_basic.t @@ -3,19 +3,19 @@ use warnings; use Class::Unload; use Data::Validate::Sanctions; -use YAML::XS qw(Dump); +use YAML::XS qw(Dump); use Path::Tiny qw(tempfile); use Test::Exception; use Test::Warnings; use Test::More; -$ENV{SANCTION_FILE} = "../share/sanctions.yml"; +$ENV{SANCTION_FILE} = "./share/sanctions.yml"; ok Data::Validate::Sanctions::is_sanctioned('NEVEROV', 'Sergei Ivanovich', -253411200), "Sergei Ivanov is_sanctioned for sure"; -ok Data::Validate::Sanctions::is_sanctioned('NEVEROV', 'Sergei Ivanovich'), "Sergei Ivanov is matched even without a birth date"; -ok !Data::Validate::Sanctions::is_sanctioned('NEVEROV', 'Sergei Ivanovich', 0), "Sergei Ivanov with incorrect dob does not match any entry"; -ok !Data::Validate::Sanctions::is_sanctioned(qw(chris down)), "Chris is a good guy (dummy name)"; -ok !Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky)), "Luke is a good boy (dummy name)"; +ok Data::Validate::Sanctions::is_sanctioned('NEVEROV', 'Sergei Ivanovich'), "Sergei Ivanov is matched even without a birth date"; +ok !Data::Validate::Sanctions::is_sanctioned('NEVEROV', 'Sergei Ivanovich', 0), "Sergei Ivanov with incorrect dob does not match any entry"; +ok !Data::Validate::Sanctions::is_sanctioned(qw(chris down)), "Chris is a good guy (dummy name)"; +ok !Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky)), "Luke is a good boy (dummy name)"; throws_ok { Data::Validate::Sanctions::set_sanction_file() } qr/sanction_file is needed/, "sanction file is required"; @@ -39,10 +39,10 @@ $tempfile->spew( lives_ok { Data::Validate::Sanctions::set_sanction_file("$tempfile"); }; is(Data::Validate::Sanctions::get_sanction_file(), "$tempfile", "get sanction file ok"); -ok !Data::Validate::Sanctions::is_sanctioned(qw(sergei ivanov)), "Sergei Ivanov is a good boy now"; -ok Data::Validate::Sanctions::is_sanctioned(qw(chris down)), "Chris is a bad boy now"; +ok !Data::Validate::Sanctions::is_sanctioned(qw(sergei ivanov)), "Sergei Ivanov is a good boy now"; +ok Data::Validate::Sanctions::is_sanctioned(qw(chris down)), "Chris is a bad boy now"; ok Data::Validate::Sanctions::is_sanctioned(qw(chris down), Date::Utility->new('1974-07-01')->epoch), "Chris is a bad boy even with birthdate"; -ok Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky)), "Luke is a bad boy without date of birth"; +ok Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky)), "Luke is a bad boy without date of birth"; ok Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky), Date::Utility->new('1996-10-10')->epoch), "Luke is a bad boy if year of birth matches"; ok !Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky), Date::Utility->new('1990-01-10')->epoch), "Luke is not sanctioned with mismatching year of birth"; @@ -53,9 +53,7 @@ $tempfile->spew( })); lives_ok { Data::Validate::Sanctions::set_sanction_file("$tempfile"); }; is(Data::Validate::Sanctions::get_sanction_file(), "$tempfile", "get sanction file ok"); -like Test::Warnings::warning { ok !Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky)) }, - qr/Content is empty for the sanction source test1. The sanctions file should be updated./, - 'Correct warnings for empty souorce content'; -ok !Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky)), "No warnings for the subsequent checks"; + +ok !Data::Validate::Sanctions::is_sanctioned(qw(Luke Lucky)), "No sanction match found with empty source"; done_testing; diff --git a/t/10_env.t b/t/02_env.t similarity index 97% rename from t/10_env.t rename to t/02_env.t index 23ea5d62..247e1d87 100644 --- a/t/10_env.t +++ b/t/02_env.t @@ -1,7 +1,7 @@ use strict; use warnings; -use YAML::XS qw(Dump); +use YAML::XS qw(Dump); use Path::Tiny qw(tempfile); use Test::Exception; use Test::Warnings; diff --git a/t/15_oo.t b/t/03_oo.t similarity index 83% rename from t/15_oo.t rename to t/03_oo.t index e9de43f4..763db459 100644 --- a/t/15_oo.t +++ b/t/03_oo.t @@ -1,16 +1,20 @@ use strict; use Class::Unload; use Data::Validate::Sanctions; -use YAML::XS qw(Dump); +use YAML::XS qw(Dump); use Path::Tiny qw(tempfile); use Test::Warnings; use Test::More; - +use Test::RedisServer; +use RedisDB; +my $redis_server; +eval { require Test::RedisServer; $redis_server = Test::RedisServer->new(conf => {port => 6379}) } + or plan skip_all => 'Test::RedisServer is required for this test'; my $validator = Data::Validate::Sanctions->new; ok $validator->is_sanctioned('NEVEROV', 'Sergei Ivanovich', -253411200), "Sergei Ivanov is_sanctioned for sure"; my $result = $validator->get_sanctioned_info('abu', 'usama', -306028800); -is $result->{matched}, 1; +is $result->{matched}, 1; is $result->{matched_args}->{dob_epoch}, -306028800; ok $result->{matched_args}->{name} =~ m/\babu\b/i and $result->{matched_args}->{name} =~ m/\busama\b/i; @@ -23,7 +27,7 @@ $result = $validator->get_sanctioned_info('Ali', 'Abu'); is $result->{matched}, 1, 'Should match because has dob_text'; $result = $validator->get_sanctioned_info('Abu', 'Salem', '1948-10-10'); -is $result->{matched}, 1; +is $result->{matched}, 1; is $result->{matched_args}->{dob_year}, 1948; ok $result->{matched_args}->{name} =~ m/\babu\b/i and $result->{matched_args}->{name} =~ m/\bsalem\b/i; @@ -84,12 +88,12 @@ $tmpb->spew( }); $validator = Data::Validate::Sanctions->new(sanction_file => "$tmpa"); -ok !$validator->is_sanctioned(qw(sergei ivanov)), "Sergei Ivanov not is_sanctioned"; -ok $validator->is_sanctioned(qw(tmpa)), "now sanction file is tmpa, and tmpa is in test1 list"; +ok !$validator->is_sanctioned(qw(sergei ivanov)), "Sergei Ivanov not is_sanctioned"; +ok $validator->is_sanctioned(qw(tmpa)), "now sanction file is tmpa, and tmpa is in test1 list"; ok !$validator->is_sanctioned("Mohammad reere yuyuy", "wqwqw qqqqq"), "is not in test1 list"; -ok $validator->is_sanctioned("Zaki", "Ahmad"), "is in test1 list - searched without dob"; -ok $validator->is_sanctioned("Zaki", "Ahmad", '1999-01-05'), 'the guy is sanctioned when dob year is matching'; -ok $validator->is_sanctioned("atom", "test", '1999-01-05'), "Match correctly with one world name in sanction list"; +ok $validator->is_sanctioned("Zaki", "Ahmad"), "is in test1 list - searched without dob"; +ok $validator->is_sanctioned("Zaki", "Ahmad", '1999-01-05'), 'the guy is sanctioned when dob year is matching'; +ok $validator->is_sanctioned("atom", "test", '1999-01-05'), "Match correctly with one world name in sanction list"; is_deeply $validator->get_sanctioned_info("Zaki", "Ahmad", '1999-01-05'), { @@ -193,4 +197,14 @@ ok $validator->is_sanctioned(qw(tmpb)), "get sanction file from ENV"; $validator = Data::Validate::Sanctions->new(sanction_file => "$tmpa"); ok $validator->is_sanctioned(qw(tmpa)), "get sanction file from args"; +subtest 'Subclass factory' => sub { + my $redis = RedisDB->new($redis_server->connect_info); + + my $validator = Data::Validate::Sanctions->new( + storage => 'redis', + connection => $redis + ); + is ref($validator), 'Data::Validate::Sanctions::Redis', 'A validator with redis storage is created'; +}; + done_testing; diff --git a/t/fetcher.t b/t/04_fetcher.t similarity index 78% rename from t/fetcher.t rename to t/04_fetcher.t index 70cf50b7..3174905c 100644 --- a/t/fetcher.t +++ b/t/04_fetcher.t @@ -4,15 +4,16 @@ use utf8; use Class::Unload; use Data::Validate::Sanctions; -use YAML::XS qw(Dump); +use YAML::XS qw(Dump); use Path::Tiny qw(tempfile); use List::Util qw(first); use Test::More; +use Test::Deep; use Test::Warnings; use Test::MockModule; use Test::Warn; use Test::MockObject; -use List::Util qw (any); +use List::Util; my %args = ( eu_url => "file://t/data/sample_eu.xml", @@ -39,22 +40,26 @@ subtest 'source url arguments' => sub { hmt_url => 'hmt.binary.com', ); - my $data; - warnings_like { - $data = Data::Validate::Sanctions::Fetcher::run(%test_args); - } - [ - qr/\bEU-Sanctions\b.*\bUser agent MockObject is hit by the url: eu.binary.com\b/, - qr/\bHMT-Sanctions\b.*\bUser agent MockObject is hit by the url: hmt.binary.com\b/, - qr/\bOFAC-Consolidated\b.*\bUser agent MockObject is hit by the url: ofac_con.binary.com\b/, - qr/\bOFAC-SDN\b.*\bUser agent MockObject is hit by the url: ofac_snd.binary.com\b/, - ], - 'Source urls are updated by params'; + my $data = Data::Validate::Sanctions::Fetcher::run(%test_args); + cmp_deeply $data, + { + 'HMT-Sanctions' => { + error => ignore(), + }, + 'OFAC-Consolidated' => { + error => ignore(), + }, + 'EU-Sanctions' => { + error => ignore(), + }, + 'OFAC-SDN' => { + error => ignore(), + }, + }, + 'All sources return errors - no content'; is $calls, 3 * 4, 'the fetcher tried thrice per source and failed finally.'; - is_deeply $data, {}, 'There is no result with invalid urls'; - }; subtest 'EU Sanctions' => sub { @@ -64,30 +69,27 @@ subtest 'EU Sanctions' => sub { warnings_like { $data = Data::Validate::Sanctions::Fetcher::run(%args, eu_url => undef); } - [qr/EU Sanctions will fail whithout eu_token or eu_url/, qr/Url is empty for EU-Sanctions/], - 'Correct warning when the EU sanctions token is missing'; + [qr/EU Sanctions will fail whithout eu_token or eu_url/], 'Correct warning when the EU sanctions token is missing'; - is $data->{$source_name}, undef, 'Result is empty as expected'; + cmp_deeply $data->{$source_name}, {error => ignore()}, 'There is an error in the result'; + like $data->{$source_name}->{error}, qr/Url is empty for EU-Sanctions/, 'Correct error for missing EU url'; - warning_like { - $data = Data::Validate::Sanctions::Fetcher::run( - %args, - eu_url => undef, - eu_token => 'ASDF' - ); - } - qr(\bEU-Sanctions\b.*\bUser agent MockObject is hit by the url: https://webgate.ec.europa.eu/fsd/fsf/public/files/xmlFullSanctionsList_1_1/content\?token=ASDF\b), - 'token is added to the default url'; - is $data->{$source_name}, undef, 'Result is empty'; - - warning_like { - $data = Data::Validate::Sanctions::Fetcher::run( - %args, - eu_url => 'http://dummy.binary.com', - eu_token => 'ASDF' - ); - } - qr(\bEU-Sanctions\b.*\bUser agent MockObject is hit by the url: http://dummy.binary.com at\b), 'token is not added to eu_url value'; + $data = Data::Validate::Sanctions::Fetcher::run( + %args, + eu_url => undef, + eu_token => 'ASDF' + ); + like $data->{$source_name}->{error}, + qr(\bUser agent MockObject is hit by the url: https://webgate.ec.europa.eu/fsd/fsf/public/files/xmlFullSanctionsList_1_1/content\?token=ASDF\b), + 'Token is added to the URL in error message'; + + $data = Data::Validate::Sanctions::Fetcher::run( + %args, + eu_url => 'http://dummy.binary.com', + eu_token => 'ASDF' + ); + like $data->{$source_name}->{error}, qr(\bUser agent MockObject is hit by the url: http://dummy.binary.com\b), + 'eu_url argument is directly used, without eu_token modification'; $data = Data::Validate::Sanctions::Fetcher::run(%args); ok $data->{$source_name}, 'EU Sanctions are loaded from the sample file'; @@ -146,8 +148,8 @@ subtest 'HMT Sanctions' => sub { $data = Data::Validate::Sanctions::Fetcher::run(%args); ok $data->{$source_name}, 'HMT Sanctions are loaded from the sample file'; - is $data->{$source_name}{updated}, 1587945600, "Sanctions update date matches the sample file"; - is scalar $data->{$source_name}{content}->@*, 23, "Number of names matches the content of the sample file"; + is $data->{$source_name}{updated}, 1587945600, "Sanctions update date matches the sample file"; + is scalar $data->{$source_name}{content}->@*, 23, "Number of names matches the content of the sample file"; is_deeply find_entry_by_name($data->{$source_name}, 'HOJATI Mohsen'), { @@ -197,8 +199,8 @@ subtest 'OFAC Sanctions' => sub { # OFAC sources have the same structure. We've created the samle sample file for both of them. ok $data->{$source_name}, 'Sanctions are loaded from the sample file'; - is $data->{$source_name}{updated}, 1587513600, "Sanctions update date matches the content of sample file"; - is scalar $data->{$source_name}{content}->@*, 6, "Number of names matches the content of the sample file"; + is $data->{$source_name}{updated}, 1587513600, "Sanctions update date matches the content of sample file"; + is scalar $data->{$source_name}{content}->@*, 6, "Number of names matches the content of the sample file"; my $dataset = $data->{$source_name}->{names_list}; @@ -256,7 +258,7 @@ sub find_entry_by_name { my @result; for my $entry ($data->{content}->@*) { - push(@result, $entry) if any { $_ eq $name } $entry->{names}->@*; + push(@result, $entry) if List::Util::any { $_ eq $name } $entry->{names}->@*; } return undef unless @result; diff --git a/t/05_sanctions_redis.t b/t/05_sanctions_redis.t new file mode 100644 index 00000000..9b7063c8 --- /dev/null +++ b/t/05_sanctions_redis.t @@ -0,0 +1,480 @@ +use strict; +use warnings; + +use Class::Unload; +use YAML; +use File::Slurp; +use Path::Tiny qw(tempfile); +use Test::Warnings; +use Test::More; +use Test::Fatal; +use Test::MockModule; +use Test::MockTime qw(set_fixed_time restore_time); +use RedisDB; +use JSON::MaybeUTF8 qw(encode_json_utf8 decode_json_utf8); +use Clone qw(clone); + +use Data::Validate::Sanctions::Redis; + +my $redis_server; +eval { require Test::RedisServer; $redis_server = Test::RedisServer->new(conf => {port => 6379}) } + or plan skip_all => 'Test::RedisServer is required for this test'; + +my $redis = RedisDB->new($redis_server->connect_info); + +my $sample_data = { + 'EU-Sanctions' => { + updated => 91, + content => [{ + names => ['TMPA'], + dob_epoch => [], + dob_year => [] + }, + { + names => ['MOHAMMAD EWAZ Mohammad Wali'], + dob_epoch => [], + dob_year => [] + }, + ], + }, + 'HMT-Sanctions' => { + updated => 150, + content => [{ + names => ['Zaki Izzat Zaki AHMAD'], + dob_epoch => [], + dob_year => [1999], + dob_text => ['other info'], + }, + ] + }, + 'OFAC-Consolidated' => { + updated => 50, + content => [{ + names => ['Atom'], + dob_year => [1999], + }, + { + names => ['Donald Trump'], + dob_text => ['circa-1951'], + }, + ] + }, + 'OFAC-SDN' => { + updated => 100, + content => [{ + names => ['Bandit Outlaw'], + place_of_birth => ['ir'], + residence => ['fr', 'us'], + nationality => ['de', 'gb'], + citizen => ['ru'], + postal_code => ['123321'], + national_id => ['321123'], + passport_no => ['asdffdsa'], + }] + }, +}; + +subtest 'Class constructor' => sub { + clear_redis(); + my $validator; + like exception { $validator = Data::Validate::Sanctions::Redis->new() }, qr/Redis connection is missing/, 'Correct error for missing redis'; + + ok $validator = Data::Validate::Sanctions::Redis->new(connection => $redis), 'Successfully created the object with redis object'; + is_deeply $validator->data, + { + 'EU-Sanctions' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'HMT-Sanctions' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'OFAC-Consolidated' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'OFAC-SDN' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + }, + 'There is no sanction data'; +}; + +subtest 'Update Data' => sub { + clear_redis(); + my $mock_fetcher = Test::MockModule->new('Data::Validate::Sanctions::Fetcher'); + my $mock_data = { + 'EU-Sanctions' => { + updated => 90, + content => [] + }, + }; + $mock_fetcher->redefine(run => sub { return clone($mock_data) }); + set_fixed_time(1500); + my $validator = Data::Validate::Sanctions::Redis->new(connection => $redis); + $validator->update_data(); + my $expected = { + 'EU-Sanctions' => { + content => [], + updated => 90, + verified => 1500, + }, + 'HMT-Sanctions' => { + content => [], + verified => 1500, + updated => 0, + error => '' + }, + 'OFAC-Consolidated' => { + content => [], + verified => 1500, + updated => 0, + error => '' + }, + 'OFAC-SDN' => { + content => [], + verified => 1500, + updated => 0, + error => '' + }, + }; + is_deeply $validator->data, $expected, 'Data is correctly loaded'; + check_redis_content('EU-Sanctions', $mock_data->{'EU-Sanctions'}, 1500); + check_redis_content('HMT-Sanctions', {}, 1500); + check_redis_content('OFAC-Consolidated', {}, 1500); + check_redis_content('OFAC-SDN', {}, 1500); + + # rewrite to redis if update (publish) time is changed + set_fixed_time(1600); + $mock_data->{'EU-Sanctions'}->{updated} = 91; + $validator->update_data(); + $expected->{'EU-Sanctions'}->{updated} = 91; + $expected->{$_}->{verified} = 1600 for keys %$expected; + is_deeply $validator->data, $expected, 'Data is loaded with new update time'; + check_redis_content('EU-Sanctions', $mock_data->{'EU-Sanctions'}, 1600, 'Redis content changed by increased update time'); + + # redis is updated with change in the number of entities, even if the publish date is the same + $mock_data = { + 'EU-Sanctions' => { + updated => 91, + content => [{ + names => ['TMPA'], + dob_epoch => [], + dob_year => [] + }, + { + names => ['MOHAMMAD EWAZ Mohammad Wali'], + dob_epoch => [], + dob_year => [] + }, + ] + }, + }; + $expected->{'EU-Sanctions'} = clone($mock_data->{'EU-Sanctions'}); + set_fixed_time(1700); + $validator->update_data(); + $expected->{$_}->{verified} = 1700 for keys %$expected; + is_deeply $validator->data, $expected, 'Data is changed with new entries, even with the same update date'; + check_redis_content('EU-Sanctions', $expected->{'EU-Sanctions'}, 1700, 'New entries appear in Redis'); + + # In case of error, content and dates are not changed + set_fixed_time(1800); + $mock_data->{'EU-Sanctions'}->{error} = 'Test error'; + $mock_data->{'EU-Sanctions'}->{updated} = 92; + $mock_data->{'EU-Sanctions'}->{content} = [1, 2, 3]; + like Test::Warnings::warning { $validator->update_data() }, qr/EU-Sanctions list update failed because: Test error/, + 'Error warning appears in logs'; + $expected->{'EU-Sanctions'}->{error} = 'Test error'; + $expected->{$_}->{verified} = 1800 for keys %$expected; + is_deeply $validator->data, $expected, 'Data is not changed if there is error'; + check_redis_content('EU-Sanctions', $expected->{'EU-Sanctions'}, 1800, 'Redis content is not changed when there is an error'); + + set_fixed_time(1850); + $validator = Data::Validate::Sanctions::Redis->new(connection => $redis); + is_deeply $validator->data->{'EU-Sanctions'}, $expected->{'EU-Sanctions'}, 'All fieds are correctly loaded form redis in constructor'; + + # All sources are updated at the same time + $mock_data = $sample_data; + $expected = clone($mock_data); + set_fixed_time(1900); + $validator->update_data(); + $expected->{$_}->{verified} = 1900 for keys %$expected; + is_deeply $validator->data, $expected, 'Data is populated from all sources'; + check_redis_content('EU-Sanctions', $mock_data->{'EU-Sanctions'}, 1900, 'EU-Sanctions error is removed with the same content and update date'); + check_redis_content('HMT-Sanctions', $mock_data->{'HMT-Sanctions'}, 1900, 'Sanction list is stored in redis'); + check_redis_content('OFAC-Consolidated', $mock_data->{'OFAC-Consolidated'}, 1900, 'Sanction list is stored in redis'); + check_redis_content('OFAC-SDN', $mock_data->{'OFAC-SDN'}, 1900, 'Sanction list is stored in redis'); + + restore_time(); + $mock_fetcher->unmock_all; +}; + +subtest 'load data' => sub { + clear_redis(); + my $validator = Data::Validate::Sanctions::Redis->new(connection => $redis); + my $expected = { + 'EU-Sanctions' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'HMT-Sanctions' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'OFAC-Consolidated' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'OFAC-SDN' => { + content => [], + verified => 0, + updated => 0, + error => '' + }}; + is_deeply $validator->data, $expected, 'Saction lists are loaded with default values when redis is empty'; + is $validator->last_updated, 0, 'Updated date is zero'; + + my $test_data = { + 'EU-Sanctions' => {}, + 'HMT-Sanctions' => { + updated => 1001, + content => [{names => ['TMPA']}], + verified => 1101, + extra_field => 1 + }, + 'OFAC-SDN' => { + updated => 1002, + content => [], + verified => 1102, + error => 'Test error' + }}; + + $expected = { + 'EU-Sanctions' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'HMT-Sanctions' => { + updated => 1001, + content => [{names => ['TMPA']}], + verified => 1101, + error => '' + }, + 'OFAC-Consolidated' => { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'OFAC-SDN' => { + updated => 1002, + content => [], + verified => 1102, + error => 'Test error' + }}; + + for my $source (keys %$test_data) { + # save data to redis + for my $field (keys $test_data->{$source}->%*) { + my $value = $test_data->{$source}->{$field}; + $value = encode_json_utf8($value) if ref $value; + $redis->hmset("SANCTIONS::$source", $field, $value); + } + } + + $validator = Data::Validate::Sanctions::Redis->new(connection => $redis); + is_deeply $validator->data->{'EU-Sanctions'}, + { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'EU sanctions list loaded with default values from Redis'; + is_deeply $validator->data->{'HMT-Sanctions'}, + { + updated => 1001, + content => [{names => ['TMPA']}], + verified => 1101, + error => '' + }, + 'HMT sanctions loaded correctly with extra field ignored'; + is_deeply $validator->data->{'OFAC-SDN'}, + { + updated => 1002, + content => [], + verified => 1102, + error => 'Test error' + }, + 'OFAC-SND loaded with correct error'; + is_deeply $validator->data->{'OFAC-Consolidated'}, + { + content => [], + verified => 0, + updated => 0, + error => '' + }, + 'Missing source OFAC-Consolodated loaded with default values'; + is $validator->last_updated, 1002, 'Update date is the maximum of the dates in all sources'; +}; + +subtest 'get sanctioned info' => sub { + # reload data freshly from the sample data + clear_redis(); + set_fixed_time(1000); + my $mock_fetcher = Test::MockModule->new('Data::Validate::Sanctions::Fetcher'); + $mock_fetcher->redefine(run => sub { return clone($sample_data) }); + my $validator = Data::Validate::Sanctions::Redis->new(connection => $redis); + $validator->update_data(); + + # create a new new validator for sanction checks. No write_redis is needed. + $validator = Data::Validate::Sanctions::Redis->new(connection => $redis); + $sample_data->{$_}->{verified} = 1000 for keys %$sample_data; + $sample_data->{$_}->{error} = '' for keys %$sample_data; + is_deeply $validator->data, $sample_data, 'Sample data is correctly loaded'; + + ok !$validator->is_sanctioned(qw(sergei ivanov)), "Sergei Ivanov not is_sanctioned"; + ok $validator->is_sanctioned(qw(tmpa)), "now sanction file is tmpa, and tmpa is in test1 list"; + ok !$validator->is_sanctioned("Mohammad reere yuyuy", "wqwqw qqqqq"), "is not in test1 list"; + ok $validator->is_sanctioned("Zaki", "Ahmad"), "is in test1 list - searched without dob"; + ok $validator->is_sanctioned("Zaki", "Ahmad", '1999-01-05'), 'the guy is sanctioned when dob year is matching'; + ok $validator->is_sanctioned("atom", "test", '1999-01-05'), "Match correctly with one world name in sanction list"; + + is_deeply $validator->get_sanctioned_info("Zaki", "Ahmad", '1999-01-05'), + { + 'comment' => undef, + 'list' => 'HMT-Sanctions', + 'matched' => 1, + 'matched_args' => { + 'dob_year' => 1999, + 'name' => 'Zaki Izzat Zaki AHMAD' + } + }, + 'Sanction info is correct'; + ok $validator->is_sanctioned("Ahmad", "Ahmad", '1999-10-10'), "is in test1 list"; + + is_deeply $validator->get_sanctioned_info("TMPA"), + { + 'comment' => undef, + 'list' => 'EU-Sanctions', + 'matched' => 1, + 'matched_args' => {'name' => 'TMPA'} + }, + 'Sanction info is correct'; + + is_deeply $validator->get_sanctioned_info('Donald', 'Trump', '1999-01-05'), + { + 'comment' => 'dob raw text: circa-1951', + 'list' => 'OFAC-Consolidated', + 'matched' => 1, + 'matched_args' => {'name' => 'Donald Trump'} + }, + "When client's name matches a case with dob_text"; + + is_deeply $validator->get_sanctioned_info('Bandit', 'Outlaw', '1999-01-05'), + { + 'comment' => undef, + 'list' => 'OFAC-SDN', + 'matched' => 1, + 'matched_args' => {'name' => 'Bandit Outlaw'} + }, + "If optional ares are empty, only name is matched"; + + my $args = { + first_name => 'Bandit', + last_name => 'Outlaw', + place_of_birth => 'Iran', + residence => 'France', + nationality => 'Germany', + citizen => 'Russia', + postal_code => '123321', + national_id => '321123', + passport_no => 'asdffdsa', + }; + + is_deeply $validator->get_sanctioned_info($args), + { + 'comment' => undef, + 'list' => 'OFAC-SDN', + 'matched' => 1, + 'matched_args' => { + name => 'Bandit Outlaw', + place_of_birth => 'ir', + residence => 'fr', + nationality => 'de', + citizen => 'ru', + postal_code => '123321', + national_id => '321123', + passport_no => 'asdffdsa', + } + }, + "All matched fields are returned"; + + for my $field (qw/place_of_birth residence nationality citizen postal_code national_id passport_no/) { + is_deeply $validator->get_sanctioned_info({%$args, $field => 'Israel'}), + {'matched' => 0}, "A single wrong field will result in mismatch - $field"; + + my $expected_result = { + 'list' => 'OFAC-SDN', + 'matched' => 1, + 'matched_args' => { + name => 'Bandit Outlaw', + place_of_birth => 'ir', + residence => 'fr', + nationality => 'de', + citizen => 'ru', + postal_code => '123321', + national_id => '321123', + passport_no => 'asdffdsa', + }, + comment => undef, + }; + + delete $expected_result->{matched_args}->{$field}; + is_deeply $validator->get_sanctioned_info({%$args, $field => undef}), $expected_result, "Missing optional args are ignored - $field"; + } + restore_time(); +}; + +sub clear_redis { + for my $key ($redis->keys('SANCTIONS::*')->@*) { + $redis->del($key); + } +} + +sub check_redis_content { + my ($source_name, $config, $verified_time, $comment) = @_; + $comment //= 'Redis content is correct'; + + my %stored = $redis->hgetall("SANCTIONS::$source_name")->@*; + $stored{content} = decode_json_utf8($stored{content}); + + is_deeply \%stored, + { + content => $config->{content} // [], + updated => $config->{updated} // 0, + error => $config->{error} // '', + verified => $verified_time, + }, + "$comment - $source_name"; +} + +done_testing; diff --git a/t/fetcher_sources.t b/t/06_fetcher_sources.t similarity index 100% rename from t/fetcher_sources.t rename to t/06_fetcher_sources.t diff --git a/xt/20_update.t b/xt/20_update.t index 81d6a32a..ce4eac51 100644 --- a/xt/20_update.t +++ b/xt/20_update.t @@ -3,7 +3,7 @@ use warnings; use Test::More; use File::Temp qw(tempfile); -use FindBin qw($Bin); +use FindBin qw($Bin); use File::stat; use Path::Tiny; use YAML::XS qw(Dump);