From e5e1c0dde84f5160665f6e87cf9c17d5430ffcff Mon Sep 17 00:00:00 2001 From: Fredrik Linder Date: Tue, 15 Apr 2014 10:55:33 -0700 Subject: [PATCH] Removed (most) compilation warnings --- rebar.config | 1 - src/uca/ux_uca_alt.erl | 8 +- src/uca/ux_uca_compress.erl | 116 +++++----- src/uca/ux_uca_decomp.erl | 11 +- src/uca/ux_uca_extract.erl | 97 ++++---- src/uca/ux_uca_sort_key_binary.erl | 19 +- src/uca/ux_uca_sort_key_binary_cs.erl | 19 +- src/uca/ux_uca_sort_key_list.erl | 17 +- src/uca/ux_uca_sort_key_uncompressed.erl | 17 +- src/uca/ux_uca_testdata.erl | 13 +- src/unidata/ux_unidata_filelist.erl | 101 +++++---- src/unidata/ux_unidata_parser.erl | 46 ++-- src/unidata/ux_unidata_parser_allkeys.erl | 68 +++--- src/unidata/ux_unidata_server.erl | 46 ++-- src/unidata/ux_unidata_store.erl | 67 +++--- src/unidata/ux_unidata_store_sup.erl | 7 +- src/utils/ux_opt_ranges.erl | 93 ++++---- src/utils/ux_ranges.erl | 16 +- src/ux_string.erl | 258 ++++++++++----------- src/ux_uca.erl | 261 +++++++++++----------- src/ux_unidata.erl | 73 +++--- src/ux_wb.erl | 108 +++++---- 22 files changed, 722 insertions(+), 740 deletions(-) diff --git a/rebar.config b/rebar.config index ac94995..0f5df8c 100644 --- a/rebar.config +++ b/rebar.config @@ -19,4 +19,3 @@ {edoc_opts, [{doclet, edown_doclet}, {src_path, ["src/"]}, {subpackages, true}]}. - diff --git a/src/uca/ux_uca_alt.erl b/src/uca/ux_uca_alt.erl index 23e4c83..b63c70f 100644 --- a/src/uca/ux_uca_alt.erl +++ b/src/uca/ux_uca_alt.erl @@ -7,13 +7,13 @@ -spec get_alternate_function(#uca_options{}, fun()) -> fun(). get_alternate_function(#uca_options{alternate='shifted', strength=4}, D) -> R = D({reassign_function, 4}), - Common = R(get_common_value), + Common = R(get_common_value), shifted_weight(Common); -get_alternate_function(C=#uca_options{alternate=Alt}, _D) -> +get_alternate_function(#uca_options{alternate=Alt}, _D) -> get_function(Alt). - + -spec get_function(Alt :: ux_uca:uca_alternate()) -> fun(). get_function(non_ignorable) -> fun non_ignorable_weight/1; @@ -141,5 +141,5 @@ shift_trimmed_weight2([_|_] = Value) -> %% @private %% Return: [_, _, _, _] -set_l4_to_value([Var, L1, L2, L3, _L4], NewL4) -> +set_l4_to_value([_Var, L1, L2, L3, _L4], NewL4) -> [L1, L2, L3, NewL4]. diff --git a/src/uca/ux_uca_compress.erl b/src/uca/ux_uca_compress.erl index 02e8da2..0b0deda 100644 --- a/src/uca/ux_uca_compress.erl +++ b/src/uca/ux_uca_compress.erl @@ -17,30 +17,30 @@ reassign_fun(Lvl, Min, OldMax) -> MinTop = NewMin + BottomSize, GapSize = MaxBottom - MinTop, true = (GapSize > 0), - + Bound = MaxBottom + (GapSize div 2), ?DBG( "~w:reassign_fun: Level ~w.~n" - " COMMON is ~w. ~n" - " MIN is ~w. ~n" - " MAX is ~w. ~n" - " BOUND is ~w. ~n" - " GAP_SIZE is ~w. ~n" - " TOP_SIZE is ~w. ~n" - " BOT_SIZE is ~w. ~n" - " MAX_BOT is ~w. ~n" - " MIN_TOP is ~w. ~n" - " OLD_MAX is ~w. ~n" - , + " COMMON is ~w. ~n" + " MIN is ~w. ~n" + " MAX is ~w. ~n" + " BOUND is ~w. ~n" + " GAP_SIZE is ~w. ~n" + " TOP_SIZE is ~w. ~n" + " BOT_SIZE is ~w. ~n" + " MAX_BOT is ~w. ~n" + " MIN_TOP is ~w. ~n" + " OLD_MAX is ~w. ~n" + , [?MODULE, Lvl, - Common, Min, Max, Bound, + Common, Min, Max, Bound, GapSize, TopSize, BottomSize, MaxBottom, MinTop, OldMax]), %% Reassign the weights in the collation element table at level n to create -%% a gap of size GAP above COMMON. Typically for secondaries or tertiaries -%% this is done after the values have been reduced to a byte range by the -%% above methods. Here is a mapping that moves weights up or down to create +%% a gap of size GAP above COMMON. Typically for secondaries or tertiaries +%% this is done after the values have been reduced to a byte range by the +%% above methods. Here is a mapping that moves weights up or down to create %% a gap in a byte range. %% w -> w + 01 - MIN, for MIN <= w < COMMON %% w -> w + FF - MAX, for COMMON < w <= MAX @@ -48,64 +48,64 @@ reassign_fun(Lvl, Min, OldMax) -> (W) when W > Common -> W + NewMax - Max end, -%% If a synthetic high weight would be less than BOUND, use a -%% sequence of high weights of the form (BOUND)..(BOUND)(MAXBOTTOM - +%% If a synthetic high weight would be less than BOUND, use a +%% sequence of high weights of the form (BOUND)..(BOUND)(MAXBOTTOM - %% remainder). - SynFn = fun(high, SynWeight, List) when SynWeight < Bound -> + SynFn = fun(high, SynWeight, List) when SynWeight < Bound -> {Remainder, NewList} = do_seq(Bound, SynWeight, List), [(MaxBottom - Remainder) | NewList]; -%% If a synthetic low weight would not be less than BOUND, use a sequence -%% of low weights of the form (BOUND-1)..(BOUND-1)(MINTOP + remainder) to +%% If a synthetic low weight would not be less than BOUND, use a sequence +%% of low weights of the form (BOUND-1)..(BOUND-1)(MINTOP + remainder) to %% express the length of the sequence. - (low, SynWeight, List) -> + (low, SynWeight, List) -> {Remainder, NewList} = do_seq(Bound - 1, SynWeight, List), [(MinTop + Remainder) | NewList] end, - -%% When generating a sort key, look for maximal sequences of m COMMON values + +%% When generating a sort key, look for maximal sequences of m COMMON values %% in a row. Let W be the weight right after the sequence. - Capacity = Max - Min, + %% Capacity = Max - Min, Result = [], Len = get_bits_len(Max), fun({to_binary, W}) -> <<<> || X <- W>>; - (get_common_value) + (get_common_value) -> Common; % for ux_uca_alt (Weights) -> do_reassign(Common, SynFn, RaFn, Weights, Result) end. - + do_seq(Val, Rem, List) when Rem >= Val -> do_seq(Val, Rem - Val, [Val|List]); -do_seq(Val, Rem, List) -> +do_seq(_Val, Rem, List) -> {Rem, List}. - + get_bits_len(Max) -> if Max =< 16#FF -> 8; Max =< 16#FFFF -> 16; - Max =< 16#FFFFFF -> 24 + Max =< 16#FFFFFF -> 24 end. %% @param W:[int()] List of weights on this lvl. %% @param R:[int()] Reversed list of resulted weights. %% @param Cnt:int() Count of repeated Commons. -%% The last step is a bit too simple, because the synthetic weights must -%% not collide with other values having long strings of COMMON weights. -%% This is done by using a sequence of synthetic weights, absorbing as -%% much length into each one as possible. -%% A value BOUND is defined between MINTOP and MAXBOTTOM. -%% The exact value for BOUND can be chosen based on the expected -%% frequency of synthetic low weights versus high weights for the +%% The last step is a bit too simple, because the synthetic weights must +%% not collide with other values having long strings of COMMON weights. +%% This is done by using a sequence of synthetic weights, absorbing as +%% much length into each one as possible. +%% A value BOUND is defined between MINTOP and MAXBOTTOM. +%% The exact value for BOUND can be chosen based on the expected +%% frequency of synthetic low weights versus high weights for the %% particular collation element table. -%% When generating a sort key, look for maximal sequences of -%% m (Cnt) COMMON values in a row. -do_reassign(Common, SynFn, RaFn, [W|WT], R) - when (W =:= Common) -> +%% When generating a sort key, look for maximal sequences of +%% m (Cnt) COMMON values in a row. +do_reassign(Common, SynFn, RaFn, [W|WT], R) + when (W =:= Common) -> {Cnt, NewWT} = do_common(W, WT, 1), Type = syn_weight_type(NewWT, Common), NewR = SynFn(Type, Cnt, R), @@ -115,18 +115,18 @@ do_reassign(Common, SynFn, RaFn, [W|WT], R) -> do_reassign(Common, SynFn, RaFn, WT, [NewW|R]); do_reassign(_Common, _SynFn, _RaFn, []=_W, R) -> lists:reverse(R). - + %% The parameter is the tail of the string after the sequence. %% -%% Let W be the weight right after the sequence. -%% If W < COMMON (or there is no W), replace the sequence by a synthetic +%% Let W be the weight right after the sequence. +%% If W < COMMON (or there is no W), replace the sequence by a synthetic %% low weight equal to (MINTOP + m). -%% If W > COMMON, replace the sequence by a synthetic high weight equal +%% If W > COMMON, replace the sequence by a synthetic high weight equal %% to (MAXBOTTOM - m). -syn_weight_type([W|_], Common) when W > Common -> +syn_weight_type([W|_], Common) when W > Common -> high; -syn_weight_type(_Str, _Common) -> +syn_weight_type(_Str, _Common) -> low. @@ -136,11 +136,11 @@ do_common(W, [WH|WT], Cnt) when WH=:=W -> do_common(_W, WT, Cnt) -> {Cnt, WT}. -get_common_value(_L = 2, Max) -> +get_common_value(_L = 2, Max) -> {32, Max}; -get_common_value(_L = 3, Max) -> +get_common_value(_L = 3, Max) -> {2, Max}; -get_common_value(_L = 4, Max) -> +get_common_value(_L = 4, Max) -> {Max+1, Max}. %get_common_value(_L = 4) -> 16#FFFF. @@ -164,9 +164,9 @@ cmp(X, Y) when X < Y -> '<'; cmp(X, X) -> '='; cmp(_, _) -> '>'. --define(_assertLower(X, Y), - {unicode:characters_to_list(io_lib:format("Is ~w < ~w?", [X, Y])), - ?_assertEqual([X, '<', Y], [X, cmp(X, Y), Y])}). +-define(_assertLower(X, Y), + {unicode:characters_to_list(io_lib:format("Is ~w < ~w?", [X, Y])), + ?_assertEqual([X, '<', Y], [X, cmp(X, Y), Y])}). binarize(Fn) -> @@ -187,12 +187,12 @@ lvl4_test_() -> %% DATA1: %% Error (key): [8427,820] greater [820,1425] %% Key1: <<0,0,158,131,0,224>> -%% Key2: <<0,0,158,0,225>> +%% Key2: <<0,0,158,0,225>> %% Arr1: [[non_variable,0,97,2,8427],[non_variable,0,124,2,820]] %% Arr2: [[non_variable,0,124,2,820],[non_variable,0,0,0,1425]] %% Error in the compression algorithm. %% Unzip Key1: [0,97,124,0,2,2] -%% Unzip Key2: [0,124,0,2] +%% Unzip Key2: [0,124,0,2] %% sort_key and compare returns different results. %% Data1: 20EB 0334 %% Data2: 0334 0591 @@ -200,7 +200,7 @@ lvl4_test_() -> %% Result (it is from eunit's output): %% ux_uca_compress:162: lvl4_test_ (Is <<131,158>> < <<158>>?)...[0.001 s] ok - + %% ux_uca_compress:reassign_fun: Level 2. %% COMMON is 32. %% MIN is 0. @@ -212,7 +212,7 @@ lvl4_test_() -> %% MAX_BOT is 66. %% MIN_TOP is 33. %% OLD_MAX is 221. -%% +%% %% =INFO REPORT==== 19-Jun-2012::13:41:39 === %% ux_uca_compress:reassign_fun: Level 3. %% COMMON is 2. @@ -225,7 +225,7 @@ lvl4_test_() -> %% MAX_BOT is 226. %% MIN_TOP is 3. %% OLD_MAX is 31. -%% +%% %% =INFO REPORT==== 19-Jun-2012::13:41:39 === %% ux_uca_compress:reassign_fun: Level 4. %% COMMON is 65502. diff --git a/src/uca/ux_uca_decomp.erl b/src/uca/ux_uca_decomp.erl index 0cbf0f0..64b1c12 100644 --- a/src/uca/ux_uca_decomp.erl +++ b/src/uca/ux_uca_decomp.erl @@ -12,13 +12,13 @@ compute(Char, List) -> L3 = ux_unidata:tertiary_weight(Char), L3Max = 16#001F, cycle(1, List, F, L3, L3Max, []). - + cycle(Pos, [H|T], Ducet, Value, Max, Acc) -> - %% Set the first two L3 values to be lookup (L3), where the lookup function - %% uses the table in Section 7.2, Tertiary Weight Table. Set the remaining - %% L3 values to MAX (which in the default table is 001F). + %% Set the first two L3 values to be lookup (L3), where the lookup function + %% uses the table in Section 7.2, Tertiary Weight Table. Set the remaining + %% L3 values to MAX (which in the default table is 001F). NewL3 = if Pos>2 -> Max; true -> Value end, @@ -30,12 +30,13 @@ cycle(Pos, [H|T], Ducet, Value, Max, Acc) -> NewAcc = fill_l3(Weight, NewL3, Acc), cycle(Pos+1, T, Ducet, Value, Max, NewAcc); -cycle(_Pos, [], _Ducet, _Value, _Max, Acc) -> +cycle(_Pos, [], _Ducet, _Value, _Max, Acc) -> ux_unidata_parser_allkeys:el_to_bin(lists:reverse(Acc)). fill_l3([H|T], NewL3, Acc) -> [Var, L1, L2, _L3, L4] = H, NewH = [Var, L1, L2, NewL3, L4], + %% @TODO: Should the next line be: [NewH | fill_l3(T, NewL3, Acc)]; fill_l3(T, NewL3, Acc); fill_l3([], _NewL3, Acc) -> Acc. diff --git a/src/uca/ux_uca_extract.erl b/src/uca/ux_uca_extract.erl index 6c6bdbe..ca1d5c1 100644 --- a/src/uca/ux_uca_extract.erl +++ b/src/uca/ux_uca_extract.erl @@ -50,7 +50,7 @@ do_extract(#uca_options { false -> W3; true -> lists:map(fun case_sensitive_hack/1, W3) end, - + % ok = check_weights(W4), {W4, S2}. @@ -69,12 +69,12 @@ check_mod(#uca_options{natural_sort=NS} = C, W1, D, S1) -> end. -check_weights(W) -> do_check_weights(W). -do_check_weights([[variable,_,_,_,_]|T]) -> - do_check_weights(T); -do_check_weights([[non_variable,_,_,_,_]|T]) -> - do_check_weights(T); -do_check_weights([]) -> ok. +%% check_weights(W) -> do_check_weights(W). +%% do_check_weights([[variable,_,_,_,_]|T]) -> +%% do_check_weights(T); +%% do_check_weights([[non_variable,_,_,_,_]|T]) -> +%% do_check_weights(T); +%% do_check_weights([]) -> ok. %% This function hides C,D,S from client. -spec do_proxy(#uca_options{}, fun(), string()) -> fun(). @@ -95,10 +95,10 @@ do_proxy(C,D,S) -> end; %% One hangul sequance was found, restart check_mod (mod_continue) -> - fun(W, Acc) -> + fun(W, Acc) -> {W1, S1} = check_mod(C, W, D, S), %% lists:reverse(Acc) ++ W1 - {lists:reverse(Acc, W1), S1} + {lists:reverse(Acc, W1), S1} end; (Result) -> {lists:reverse(Result),S} end. @@ -137,10 +137,10 @@ case_invert(L3) -> -spec case_sensitive_hack(uca_elem()) -> uca_elem(). case_sensitive_hack([Var,L1,L2,L3,L4]) -> [Var,L3,L2,L1,L4]. - + % Hack for numbers. -has_mod([[_Var,L1|_]|T], _NS=true) +has_mod([[_Var,L1|_]|_T], _NS=true) when ?IS_L1_OF_DECIMAL(L1) -> true; has_mod([[_Var,L1|_]|_], _NS) @@ -181,12 +181,12 @@ has_mod([], _NS) -> % Hack for Hangul. -spec mod_weights(fun(), uca_array(), boolean(), uca_array()) -> result(). % Hack for numbers. -mod_weights(E, [[Var,L1|LOther]=H|T], _NS=true, Acc) +mod_weights(E, [[Var,L1|LOther]=_H|T], _NS=true, Acc) when ?IS_L1_OF_DECIMAL(L1) -> F = fun(W) -> [Var,W|LOther] end, % define F. Num = ?COL_WEIGHT_TO_DECIMAL(L1), do_decimal(E, F, Num, T, Acc); -mod_weights(E, [[Var,L1|_]=H|T], _NS, Acc) +mod_weights(E, [[_Var,L1|_]=H|T], _NS, Acc) when ?IS_L1_OF_HANGUL_L(L1) -> do_hangul(E, l, T, [H|Acc]); mod_weights(E, [H|T], NS, Acc) -> @@ -208,7 +208,7 @@ mod_weights(E, [], _NS, Acc) -> -spec do_decimal(fun(), fun(), boolean(), uca_array(), uca_array()) -> result(). do_decimal(E, F, N, [[_,0|_]=H|T]=_W, Acc) -> do_decimal(E, F, N, T, [H|Acc]); % skip an ignorable element. -do_decimal(E, F, N, [[_,L1|_]=H|T]=_W, Acc) +do_decimal(E, F, N, [[_,L1|_]=_H|T]=_W, Acc) when ?IS_L1_OF_DECIMAL(L1) -> NewN = (N * 10) + ?COL_WEIGHT_TO_DECIMAL(L1), ?DBG("old ~w; new ~w~n", [N, NewN]), @@ -241,6 +241,7 @@ do_decimal(E, F, N, W, Acc) -> %% @end decimal_result(F, N, Acc) -> NewAcc = [F(16#FFFE), F(1)|Acc], + %% @TODO: Should the next line be: do_decimal_result(F, N, NewAcc). do_decimal_result(F, N, Acc). -spec do_decimal_result(fun(), integer(), uca_array()) -> uca_array(). @@ -295,7 +296,7 @@ do_hangul(E, _Mod, W, Acc) -> % L %% @private -spec hangul_result(fun(), uca_array(), uca_array()) -> result(). -hangul_result(E, T, Acc) -> +hangul_result(E, _T, Acc) -> TermWeight = E(term), E([TermWeight | Acc]). @@ -308,7 +309,7 @@ hangul_result_and_continue(E, W, Acc) -> %% Step 0: try extract derived weights. %% @private -%% @param Str:string() String +%% @param Str:string() String %% @param D::fun() Ducet Function -spec do_extract0(string(), fun()) -> result(). @@ -325,21 +326,21 @@ do_extract0([H], D) -> % Last Char {W, []}; _ -> {[], []} - end; + end; do_extract0([H|T]=S, DFn) -> - % Max ccc among ccces of skipped chars beetween the starter char - % and the processed char. If there are no skipped chars, then + % Max ccc among ccces of skipped chars beetween the starter char + % and the processed char. If there are no skipped chars, then % Ccc1=false. - OldCCC = false, - Key = [], + OldCCC = false, + Key = [], Skipped = [], LFn = ducet_lookup(DFn), CFn = ux_unidata:ccc(skip_check), MFn = get_more(LFn, CFn), Res = false, - + case do_extract1(S, MFn, Key, OldCCC, Skipped, Res) of {result, Key2, T2} -> W = DFn(Key2), @@ -353,20 +354,20 @@ do_extract0([H|T]=S, DFn) -> %% @param S:string() String %% Res is a last good Key. --spec do_extract1(string(), fun(), string(), ux_ccc()|false, - string(), uca_array()) -> +-spec do_extract1(string(), fun(), string(), ux_ccc()|false, + string(), uca_array()) -> {result,string(),string()}|not_found. -do_extract1([H|T]=S, MFn, Key, OldCCC, Skipped, Res) +do_extract1([H|T]=S, MFn, Key, OldCCC, Skipped, Res) when is_list(Skipped) -> NewKey = [H|Key], case MFn(NewKey, OldCCC) of - {false, _NewCCC} when Res =:= more -> + {false, _NewCCC} when Res =:= more -> more_error; - {false, NewCCC} -> + {false, NewCCC} -> NewSkipped = [H|Skipped], do_extract1(T, MFn, Key, NewCCC, NewSkipped, Res); - {true, NewCCC} -> + {true, NewCCC} -> CCC = select_ccc(OldCCC, NewCCC), ?DBG("selected ccc is ~w.~n", [CCC]), do_extract1(T, MFn, NewKey, CCC, Skipped, NewKey); @@ -377,7 +378,7 @@ do_extract1([H|T]=S, MFn, Key, OldCCC, Skipped, Res) {maybe, NewCCC} -> CCC = select_ccc(OldCCC, NewCCC), case do_extract1(T, MFn, NewKey, CCC, Skipped, more) of - more_error -> + more_error -> NewSkipped = [H|Skipped], do_extract1(T, MFn, Key, NewCCC, NewSkipped, Res); Return -> Return @@ -391,15 +392,15 @@ do_extract1([H|T]=S, MFn, Key, OldCCC, Skipped, Res) bad_ccc -> {result, do_extract1_return(Res), lists:reverse(Skipped, S)} end; - - + + do_extract1([]=_S, _MFn, _Key, _OldCCC, Skipped, _Res=more) when is_list(Skipped) -> more_error; do_extract1([]=_S, _MFn, _Key, _OldCCC, Skipped, _Res=false) when is_list(Skipped) -> not_found; -do_extract1([]=_S, _MFn, _Key, _OldCCC, Skipped, Res) +do_extract1([]=_S, _MFn, _Key, _OldCCC, Skipped, Res) when is_list(Skipped) -> {result, do_extract1_return(Res), lists:reverse(Skipped)}. @@ -418,10 +419,10 @@ do_extract1([]=_S, _MFn, _Key, _OldCCC, Skipped, Res) % Base 3: FBC0 Any other code point % Range 3: Ideographic AND NOT Unified_Ideograph % ----------------------------------------------------------------------------- -do_implicit(H) +do_implicit(H) when ?CHAR_IS_UNIFIED_IDEOGRAPH(H) -> if - (?CHAR_IS_CJK_COMPATIBILITY_IDEOGRAPH(H) + (?CHAR_IS_CJK_COMPATIBILITY_IDEOGRAPH(H) or ?CHAR_IS_CJK_UNIFIED_IDEOGRAPH(H)) -> implicit_weight(H, 16#FB40); true -> @@ -431,7 +432,7 @@ do_implicit(H) do_implicit(H) -> implicit_weight(H, 16#FBC0). - + %% After skiping a character, we set OldCCC = NewCCC. @@ -440,7 +441,7 @@ select_ccc(_OldCCC=false, _NewCCC) -> false; select_ccc(_OldCCC, NewCCC) -> NewCCC. - + -spec do_extract1_return(string()) -> string(). do_extract1_return(Res) -> lists:reverse(Res). @@ -458,7 +459,7 @@ ducet_lookup(D) -> - + -spec get_more(fun(), fun()) -> term(). get_more(LFn, CFn) -> @@ -466,7 +467,7 @@ get_more(LFn, CFn) -> case CFn(H) of NewCCC when OldCCC =:= false; OldCCC=/=0, OldCCC - ?DBG("ccc is ok. OldCCC is ~w. NewCCC is ~w. ~n", + ?DBG("ccc is ok. OldCCC is ~w. NewCCC is ~w. ~n", [OldCCC, NewCCC]), Status = LFn(K), ?DBG("Status is ~w. ~n", [Status]), @@ -475,24 +476,24 @@ get_more(LFn, CFn) -> NewCCC when OldCCC =:= NewCCC, OldCCC =/= 0 -> ?DBG("Char is blocked. CCC is ~w. ~n", [OldCCC]), {false, NewCCC}; % blocked - NewCCC -> - ?DBG("Bad CCC. OldCCC is ~w. NewCCC is ~w ~n", - [OldCCC, NewCCC]), + _NewCCC -> + ?DBG("Bad CCC. OldCCC is ~w. NewCCC is ~w ~n", + [OldCCC, _NewCCC]), bad_ccc end end. - -% Note: A non-starter in a string is called blocked if there is another -% non-starter of the same canonical combining class or zero between + +% Note: A non-starter in a string is called blocked if there is another +% non-starter of the same canonical combining class or zero between % it and the last character of canonical combining class 0. -%% @doc 7.1.3 Implicit Weights +%% @doc 7.1.3 Implicit Weights %% The result of this process consists of collation elements that are sorted in %% code point order, that do not collide with any explicit values in the table, -%% and that can be placed anywhere (for example, at BASE) with respect to the +%% and that can be placed anywhere (for example, at BASE) with respect to the %% explicit collation element mappings. By default, implicit mappings are given %% higher weights than all explicit collation elements. %% @end @@ -500,7 +501,5 @@ get_more(LFn, CFn) -> implicit_weight(CP, BASE) when is_integer(CP) and is_integer(BASE) -> AAAA = BASE + (CP bsr 15), BBBB = (CP band 16#7FFF) bor 16#8000, - [[non_variable, AAAA, 32, 2, 0], + [[non_variable, AAAA, 32, 2, 0], [non_variable, BBBB, 0, 0, 0]]. % reversed - - diff --git a/src/uca/ux_uca_sort_key_binary.erl b/src/uca/ux_uca_sort_key_binary.erl index eb076c3..908641c 100644 --- a/src/uca/ux_uca_sort_key_binary.erl +++ b/src/uca/ux_uca_sort_key_binary.erl @@ -3,10 +3,10 @@ -import(ux_uca, [sort_array/2]). -import(ux_uca_utils, [ - do_alt/2, - get_ducet/0, - get_options/0, - split_levels/3, + do_alt/2, + get_ducet/0, + get_options/0, + split_levels/3, get_reassign_function/2]). -include("ux.hrl"). @@ -37,17 +37,17 @@ do_sort_key1(S, B, [WH|WT], D, A, R, K) -> _ -> do_sort_key1(S, B, WT, D, NewA, R, K) end; -do_sort_key1(S, B, [], D, A, R, K) +do_sort_key1(S, B, [], D, _A, R, K) when (S > 1) -> W = lists:reverse(R), L = 2, % Level - WL = [], + %% WL = [], NewK = [0|K], RevK = lists:reverse(NewK), Fn = get_reassign_function(D, 1), BinK = Fn({to_binary, RevK}), do_sort_key2(S, B, L, W, D, BinK); -do_sort_key1(S, _B, [], D, _A, _R, K) -> +do_sort_key1(_S, _B, [], D, _A, _R, K) -> Fn = get_reassign_function(D, 1), RevK = lists:reverse(K), Fn({to_binary, RevK}). % Return result @@ -59,11 +59,11 @@ do_sort_key2(S, B, L, W, D, K) -> Fn = get_reassign_function(D, L), ReassignW = Fn(LvlW), case RemW of - _ when RemW=:=[]; S= + _ when RemW=:=[]; S= BinW = Fn({to_binary, ReassignW}), <>; - [_|_] -> + [_|_] -> % Add a delimeter. BinW = Fn({to_binary, ReassignW}), Delim = Fn({to_binary, [0]}), @@ -71,4 +71,3 @@ do_sort_key2(S, B, L, W, D, K) -> do_sort_key2(S, B, L+1, RemW, D, NewK) end. - diff --git a/src/uca/ux_uca_sort_key_binary_cs.erl b/src/uca/ux_uca_sort_key_binary_cs.erl index fb81580..2dcab67 100644 --- a/src/uca/ux_uca_sort_key_binary_cs.erl +++ b/src/uca/ux_uca_sort_key_binary_cs.erl @@ -1,13 +1,13 @@ -%%% This module is a variant of ux_uca_sort_key_binary +%%% This module is a variant of ux_uca_sort_key_binary %%% for case sensative collation. -module(ux_uca_sort_key_binary_cs). -export([sort_key/2]). -import(ux_uca, [sort_array/2]). -import(ux_uca_utils, [ - do_alt/2, - get_ducet/0, - get_options/0, - split_levels/3, + do_alt/2, + get_ducet/0, + get_options/0, + split_levels/3, get_reassign_function/2]). -include("ux.hrl"). @@ -38,11 +38,11 @@ do_sort_key1(S, B, [WH|WT], D, A, R, K) -> _ -> do_sort_key1(S, B, WT, D, NewA, R, K) end; -do_sort_key1(S, B, [], D, A, R, K) +do_sort_key1(S, B, [], D, _A, R, K) when (S > 1) -> W = lists:reverse(R), L = 2, % Level - WL = [], + %% WL = [], NewK = [0|K], RevK = lists:reverse(NewK), Fn = get_reassign_function(D, 3), @@ -64,12 +64,12 @@ do_sort_key2(S, B, L, W, D, K) -> % So, now we use to_binary for L1 on L3, and v.v. BinFn = get_reassign_function(D, case L of 3 -> 1; _ -> L end), case RemW of - _ when RemW=:=[]; S= + _ when RemW=:=[]; S= RevW = lists:reverse(ReassignW), BinW = BinFn({to_binary, RevW}), <>; - [_|_] -> + [_|_] -> % Add a delimeter. NewW = [0|ReassignW], RevW = lists:reverse(NewW), @@ -78,4 +78,3 @@ do_sort_key2(S, B, L, W, D, K) -> do_sort_key2(S, B, L+1, RemW, D, NewK) end. - diff --git a/src/uca/ux_uca_sort_key_list.erl b/src/uca/ux_uca_sort_key_list.erl index 5b9fe52..703312b 100644 --- a/src/uca/ux_uca_sort_key_list.erl +++ b/src/uca/ux_uca_sort_key_list.erl @@ -3,10 +3,10 @@ -import(ux_uca, [sort_array/2]). -import(ux_uca_utils, [ - do_alt/2, - get_ducet/0, - get_options/0, - split_levels/3, + do_alt/2, + get_ducet/0, + get_options/0, + split_levels/3, get_reassign_function/2]). -include("ux.hrl"). @@ -36,13 +36,13 @@ do_sort_key1(S, B, [WH|WT], D, A, R, K) -> _ -> do_sort_key1(S, B, WT, D, NewA, R, K) end; -do_sort_key1(S, B, [], D, A, R, K) +do_sort_key1(S, B, [], D, _A, R, K) when (S > 1) -> W = lists:reverse(R), L = 2, % Level - WL = [], + %% WL = [], do_sort_key2(S, B, L, W, D, [0|K]); -do_sort_key1(S, B, [], D, A, R, K) -> +do_sort_key1(_S, _B, [], _D, _A, _R, K) -> K. % Return result %% L::int() Level @@ -54,8 +54,7 @@ do_sort_key2(S, B, L, W, D, K) -> NewK = lists:reverse(ReassignW, K), case NewW of [] -> NewK; - _ when S= NewK; [_|_] -> do_sort_key2(S, B, L+1, NewW, D, [0|NewK]) end. - diff --git a/src/uca/ux_uca_sort_key_uncompressed.erl b/src/uca/ux_uca_sort_key_uncompressed.erl index d26ff38..987cbc0 100644 --- a/src/uca/ux_uca_sort_key_uncompressed.erl +++ b/src/uca/ux_uca_sort_key_uncompressed.erl @@ -3,10 +3,10 @@ -import(ux_uca, [sort_array/2]). -import(ux_uca_utils, [ - do_alt/2, - get_ducet/0, - get_options/0, - split_levels/3, + do_alt/2, + get_ducet/0, + get_options/0, + split_levels/3, get_reassign_function/2]). -include("ux.hrl"). @@ -36,13 +36,13 @@ do_sort_key1(S, B, [WH|WT], A, R, K) -> _ -> do_sort_key1(S, B, WT, NewA, R, K) end; -do_sort_key1(S, B, [], A, R, K) +do_sort_key1(S, B, [], _A, R, K) when (S > 1) -> W = lists:reverse(R), L = 2, % Level - WL = [], + %% WL = [], do_sort_key2(S, B, L, W, [0|K]); -do_sort_key1(S, _B, [], A, R, K) -> +do_sort_key1(_S, _B, [], _A, _R, K) -> K. % Return result %% L::int() Level @@ -52,8 +52,7 @@ do_sort_key2(S, B, L, W, K) -> NewK = lists:reverse(LvlW, K), case NewW of [] -> NewK; - _ when S= NewK; [_|_] -> do_sort_key2(S, B, L+1, NewW, [0|NewK]) end. - diff --git a/src/uca/ux_uca_testdata.erl b/src/uca/ux_uca_testdata.erl index 3b80224..4bebc70 100644 --- a/src/uca/ux_uca_testdata.erl +++ b/src/uca/ux_uca_testdata.erl @@ -11,11 +11,11 @@ read_shifted() -> read_shifted_(Fd, Num, Acc) -> case read_line(Fd, Num) of - {RawString, [], NewNum} -> + {_RawString, [], NewNum} -> read_shifted_(Fd, NewNum, Acc); - {RawString, Points, NewNum} -> + {_RawString, Points, NewNum} -> read_shifted_(Fd, NewNum, [Points|Acc]); - eof -> lists:reverse(Acc) + eof -> lists:reverse(Acc) end. @@ -47,9 +47,8 @@ read_line(Fd, StrNum) -> {Data, Res, StrNum + 1} % {FullStr, Codepaints} catch - error:Reason -> -% io:format(user, "~w: Data=~w ~n", [Reason, Data]), - read_line(Fd, StrNum + 1) + error:_Reason -> + %% io:format(user, "~w: Data=~w ~n", [Reason, Data]), + read_line(Fd, StrNum + 1) end end. - diff --git a/src/unidata/ux_unidata_filelist.erl b/src/unidata/ux_unidata_filelist.erl index bce76b3..c388b00 100644 --- a/src/unidata/ux_unidata_filelist.erl +++ b/src/unidata/ux_unidata_filelist.erl @@ -1,22 +1,21 @@ -%%% @doc Key-value store for the list of the servers which +%%% @doc Key-value store for the list of the servers which %%% serve the unidata files. %%% @private -module(ux_unidata_filelist). -include("ux.hrl"). -% OTP +% OTP -export([start_link/0]). --export([init/1, terminate/2, handle_call/3, handle_info/2]). +-behavior(gen_server). +-export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2, code_change/3]). % Inter-module exports -export([reg_pid/2, file_owner/1]). % Unidata API --export([set_source/4, set_source/2, +-export([set_source/4, set_source/2, get_source/2, get_source/1, get_source_from/2]). --behavior(gen_server). - -record(state, { key2server :: dict() }). @@ -44,13 +43,13 @@ terminate(_Reason, _LoopData) -> % Key is a combination of filename and fileoptions. handle_call({reg_pid, Key, StorePid}, _From, State = #state{key2server = K2S}) -> erlang:monitor(process, StorePid), - ?DBG("~w: Registrate a new process ~w with the key ~w. ~n", + ?DBG("~w: Registrate a new process ~w with the key ~w. ~n", [?MODULE, StorePid, Key]), - + {Reply, K2S_2} = case dict:is_key(Key, K2S) of - false -> + false -> {ok, dict:store(Key, StorePid, K2S)}; - true -> + true -> error_logger:error_msg("~w: The key ~w is already registred. ~w", [?MODULE, Key]), {{error, key_already_registred}, K2S} @@ -62,18 +61,24 @@ handle_call({get_pid, Key}, _From, State = #state{key2server = K2S}) -> Reply = dict:find(Key, K2S), % {ok, Value} or error {reply, Reply, State}. +handle_cast(_Msg, State) -> + {noreply, State}. + % Server is dead, unregister it. % Delete pid from the dict. FromPid is a pid of ux_unidata_store server. handle_info({'DOWN', _Ref, process, ServerPid, _Reason}, State = #state{key2server = K2S}) -> - ?DBG("~w: Delete Pid = ~w from the dictionary. ~n", + ?DBG("~w: Delete Pid = ~w from the dictionary. ~n", [?MODULE, FromPid]), K2S_2 = dict:filter(fun(_K, V) -> V =/= ServerPid end, K2S), State_2 = State#state{key2server = K2S_2}, {noreply, State_2}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + %% %% API @@ -98,11 +103,11 @@ handle_info({'DOWN', _Ref, process, ServerPid, _Reason}, set_source('node', Parser, Types, FileName) -> Key = {Parser, Types, FileName}, ux_unidata_server:set_default(Key); - + set_source(Level, Parser, Types, FileName) -> Key = {Parser, Types, FileName}, ClientPid = case Level of - 'process' -> + 'process' -> self(); 'application' -> % We unload data, when the application die. @@ -120,16 +125,16 @@ set_source(Level, Parser, Types, FileName) -> ('skip_check') -> case ets:info(Ets, 'owner') of - undefined -> + undefined -> set_source(process, Parser, [Type], FileName), - NewFun = get_source(Parser, Type); + _NewFun = get_source(Parser, Type); _ -> Fun end; %% For ux_unidata_server. Check ETS before return value. ('test') -> case ets:info(Ets, 'owner') of - 'undefined' -> + 'undefined' -> false; _ -> true end; @@ -137,19 +142,19 @@ set_source(Level, Parser, Types, FileName) -> %% Get an ETS table (need for CLDR). ('get_table') -> case ets:info(Ets, 'owner') of - 'undefined' -> + 'undefined' -> set_source('node', Parser, [Type], FileName), NewFun = get_source(Parser, Type), NewFun('get_table'); - _ -> + _ -> Ets end; ('reload') -> case ets:info(Ets, 'owner') of - 'undefined' -> + 'undefined' -> set_source('node', Parser, [Type], FileName), - NewFun = get_source(Parser, Type), + _NewFun = get_source(Parser, Type), ok; _ -> ok end; @@ -159,41 +164,41 @@ set_source(Level, Parser, Types, FileName) -> try Fun(Val) catch - error:badarg -> + error:badarg -> case ets:info(Ets) of - 'undefined' -> + 'undefined' -> set_source(Level, Parser, [Type], FileName), NewFun = get_source(Parser, Type), - NewFun(Val) + NewFun(Val) end - end + end end, %% of DataSourceFun % Set an upgrade trigger. {{Parser, Type}, DataSourceFun} - + end, get_funs(Key, ClientPid)), - ?DBG("~w: Loaded funs: ~w. ~n", + ?DBG("~w: Loaded funs: ~w. ~n", [?MODULE, Funs]), case Level of - 'process' -> + 'process' -> % Put to the process dictionary. set_proc_dict(Funs); 'application' -> - {ok, AppName2} = application:get_application(), + {ok, AppName2} = application:get_application(), set_app_env(AppName2, Funs) end, ok. %% This is a short form of function. -set_source(Level, {Parser, Types, Filename} = _Key) -> +set_source(Level, {Parser, Types, Filename} = _Key) -> set_source(Level, Parser, Types, Filename). %% @doc Return registred fun. -%% Check: the dict of client process, then application enviroment, +%% Check: the dict of client process, then application enviroment, %% then try get the default value from the server. get_source(Parser, Type) -> Value = {Parser, Type}, @@ -208,7 +213,7 @@ get_source(Parser, Type) -> %% Step 3: Use defaults. get_source(Value) -> case get_source_from(process, Value) of % step 1 - 'undefined' -> + 'undefined' -> case get_source_from(application, Value) of % step 2 'undefined' -> get_source_from('node', Value); % step 3 Fun -> Fun @@ -230,12 +235,12 @@ get_source_from('node', Value) -> undefined -> ux:start(), ux_unidata_server:get_default(Value); - _ -> - ux_unidata_server:get_default(Value) + _ -> + ux_unidata_server:get_default(Value) end. - - - + + + %% Return the list of functions from the server. -spec get_funs(Key::{Parser::atom(), Types::[atom()], FileName::string()}, pid()) -> [{Type::atom(), Ets::integer(), fun()}]. @@ -244,15 +249,15 @@ get_funs({_,Types,_} = Key, ClientPid) -> ux_unidata_store:get_funs(ServerPid, Types). set_app_env(Name, [{Key, Val}|Tail]) -> - ?DBG("~w: Set a application enviroment variable ~w::~w to ~w. ~n", + ?DBG("~w: Set a application enviroment variable ~w::~w to ~w. ~n", [?MODULE, Name, Key, Val]), application:set_env(Name, Key, Val), set_app_env(Name, Tail); set_app_env(_Name, []) -> ok. - - + + set_proc_dict([{Key, Val}|Tail]) -> - ?DBG("~w: Put the value to the process dictionary: ~w::~w to ~w. ~n", + ?DBG("~w: Put the value to the process dictionary: ~w::~w to ~w. ~n", [?MODULE, self(), Key, Val]), erlang:put(Key, Val), set_proc_dict(Tail); @@ -263,21 +268,21 @@ set_proc_dict([]) -> ok. get_application_pid(Name) -> AInfo = application:info(), {'running', R} = lists:keyfind('running', 1, AInfo), - {Name, Pid} = lists:keyfind(Name, 1, R). + {Name, _Pid} = lists:keyfind(Name, 1, R). -%% Try to get a pid of the owner of an ETS table with the UNIDATA +%% Try to get a pid of the owner of an ETS table with the UNIDATA %% from the Key-file. %% Also, try to monitor ClientPid on the server. get_pid(Key, ClientPid) when is_pid(ClientPid) -> case file_owner(key_to_id(Key)) of - error -> + error -> {ok, StoreServerPid} = ux_unidata_store_sup:read_file(Key, ClientPid), StoreServerPid; - + % Server is already running. - {ok, StoreServerPid} when is_pid(StoreServerPid) -> - ux_unidata_store:monitor_client_process(StoreServerPid, ClientPid), + {ok, StoreServerPid} when is_pid(StoreServerPid) -> + ux_unidata_store:monitor_client_process(StoreServerPid, ClientPid), ux_unidata_store:check_types(StoreServerPid, key_to_types(Key)), StoreServerPid end. @@ -288,9 +293,9 @@ get_pid(Key, ClientPid) when is_pid(ClientPid) -> file_owner(FileName) -> gen_server:call(?MODULE, {get_pid, FileName}). -%% Used only by ux_unidata_store:init/1. +%% Used only by ux_unidata_store:init/1. %% Don't use this function from user code. -%% Throws {badmatch,{error,key_already_registred}} if self() is already +%% Throws {badmatch,{error,key_already_registred}} if self() is already %% registred. reg_pid(Key, StoreServerPid) when is_pid(StoreServerPid) -> ok = gen_server:call(?MODULE, {reg_pid, key_to_id(Key), StoreServerPid}). diff --git a/src/unidata/ux_unidata_parser.erl b/src/unidata/ux_unidata_parser.erl index ef8a15c..a5f193b 100644 --- a/src/unidata/ux_unidata_parser.erl +++ b/src/unidata/ux_unidata_parser.erl @@ -9,7 +9,7 @@ -export([check_types/2]). %% Helpers --export([split/2, hex_to_int/1, from_hex/1, +-export([split/2, hex_to_int/1, from_hex/1, delete_spaces/1, delete_spaces/2, delete_comments/1]). %% For ux_unidata -export([open_file/1]). @@ -20,7 +20,7 @@ -get_env({ParserType, Types, FileName} = File) -> +get_env({ParserType, _Types, _FileName} = File) -> Mod = filetype_to_module(ParserType), try Mod:bootstrap(File) @@ -30,7 +30,7 @@ get_env({ParserType, Types, FileName} = File) -> set_env(_File, 'undefined'=_Env) -> ok; -set_env({ParserType, Types, FileName} = File, Env) -> +set_env({ParserType, _Types, _FileName} = File, Env) -> Mod = filetype_to_module(ParserType), Mod:init(File, Env). @@ -52,8 +52,8 @@ check_filename(FileName) -> % File exists? case file:read_file_info(FileName) of {ok, _Info} -> ok; - Error -> - error_logger:error_msg(?MODULE_STRING ++ ": File ~s not found.", [FileName]), + Error -> + error_logger:error_msg(?MODULE_STRING ++ ": File ~s not found.", [FileName]), erlang:error(Error) end, ok. @@ -105,7 +105,7 @@ open_file(FileName) -> "zg." ++ _List -> % is .gz? file:open(FileName, [read, compressed]); _ -> - file:open(FileName, [read]) + file:open(FileName, [read]) end, Fd catch @@ -142,7 +142,7 @@ do_check_types([], _) -> true; do_check_types([_|_], _) -> false. - + -spec create_tables([atom()]) -> [{atom(), integer()}]. create_tables(DataTypes) -> do_create_tables(DataTypes, []). @@ -161,11 +161,11 @@ filetype_to_module(Type) -> list_to_atom("ux_unidata_parser_" read_file({Fd, Ets, Mod} = State) -> case file:read_line(Fd) of - {ok, []} -> + {ok, []} -> read_file(State); {ok, Line} -> case Mod:parse(delete_nr(delete_comments(Line))) of - skip -> + skip -> read_file(State); {ok, Val} -> ok = write_to_ets(Ets, lists:sort(Val)), @@ -173,14 +173,14 @@ read_file({Fd, Ets, Mod} = State) -> end; eof -> ok end. - + -spec delete_comments(string()) -> string(). delete_comments(Line) -> lists:reverse(do_delete_comments(Line, [])). do_delete_comments([], Acc) -> Acc; do_delete_comments([$# | _], Acc) -> Acc; -do_delete_comments([H|T], Acc) -> +do_delete_comments([H|T], Acc) -> do_delete_comments(T, [H|Acc]). -spec write_to_ets([{atom(), integer()}], [{atom(), tuple()}]) -> ok. @@ -209,15 +209,15 @@ do_split(Char, [Head|Tail], Acc1, Acc2) -> do_split(Char, Tail, [Head|Acc1], Acc2). hex_to_int(Code) -> - case io_lib:fread("~16u", Code) of + case io_lib:fread("~16u", Code) of {ok, [Int], []} -> Int; _ -> false end. -from_hex([$<|Str]) -> +from_hex([$<|Str]) -> SubStr = string:sub_string(Str, string:chr(Str, $>)+1), from_hex(SubStr); -from_hex(Str) -> +from_hex(Str) -> lists:map(fun hex_to_int/1, string:tokens(Str, " ")). delete_spaces(Str) -> delete_spaces(Str, $ ). @@ -236,11 +236,11 @@ expand_table(Table) -> {From, To} = Key -> [El] = ets:lookup(Table, Key), ets:delete(Table, Key), - + do_expand(Table, El, From, To), expand_table_next(Table, Key); Key -> - expand_table_next(Table, Key) + expand_table_next(Table, Key) end, ets:safe_fixtable(Table, false), ok. @@ -252,11 +252,11 @@ expand_table_next(Table, Prev) -> {From, To} = Key -> [El] = ets:lookup(Table, Key), ets:delete(Table, Key), - + do_expand(Table, El, From, To), expand_table_next(Table, Key); Key -> - expand_table_next(Table, Key) + expand_table_next(Table, Key) end. do_expand(Table, El, From, To) when From =< To -> @@ -272,7 +272,7 @@ do_expand(_, _, _, _) -> ok. %% Expand table with two colums: [{Key, Value} or {{From, To}, Value}]. expand_fun(Table, DefaultValue) -> List = ets:tab2list(Table), - fun(V) -> + fun(V) -> case ux_ranges:search(List, V) of false -> DefaultValue; Result -> Result @@ -349,13 +349,13 @@ expand_meta_fun(Table, DefaultValue) -> Body = do_fun_def(Name, DefaultValue), NewBody = do_expand_fun(Table, Name, Body, DefaultValue), - + {ok, Value} = Handler(NewBody), Value. --spec do_expand_fun(Table::integer(), Name::atom(), Body::string(), DefaultValue::term()) +-spec do_expand_fun(Table::integer(), Name::atom(), Body::string(), DefaultValue::term()) -> string(). do_expand_fun(Table, Name, Body, DefaultValue) -> case ets:first(Table) of @@ -368,7 +368,7 @@ do_expand_fun(Table, Name, Body, DefaultValue) -> do_expand_fun_next(Table, Name, NewBody, DefaultValue, Key) end. --spec do_expand_fun_next(Table::integer(), Name::atom(), Body::string(), +-spec do_expand_fun_next(Table::integer(), Name::atom(), Body::string(), DefaultValue::term(), Prev::term()) -> string(). do_expand_fun_next(Table, Name, Body, DefaultValue, Prev) -> case ets:next(Table, Prev) of @@ -486,7 +486,7 @@ expand_fun_test_() -> ets:insert(T, {{4,10}, interval}), {T, expand_fun(T, default)} - + end, fun ({T, _F}) -> ets:delete(T) end, fun ({_T, F}) -> diff --git a/src/unidata/ux_unidata_parser_allkeys.erl b/src/unidata/ux_unidata_parser_allkeys.erl index d9e61ec..6d43367 100644 --- a/src/unidata/ux_unidata_parser_allkeys.erl +++ b/src/unidata/ux_unidata_parser_allkeys.erl @@ -10,7 +10,7 @@ , after_parse/1 % comment to disable post processing ]). -%% For ux_uca_decomp +%% For ux_uca_decomp -export([el_to_bin/1]). @@ -24,9 +24,9 @@ parse(In) -> OutEl = parse_el(ux_unidata_parser:delete_spaces(Element)), %io:format("String: ~ts, From reversed: ~w, To: ~w~n", [In, InEl, OutEl]), - Res = case InEl of - [] -> skip; - _ -> {InEl, OutEl} + Res = case InEl of + [] -> skip; + _ -> {InEl, OutEl} end, {ok, [{ducet, Res} @@ -39,7 +39,7 @@ after_parse(Ets) -> do_after(Ets), ok. -get_function(ducet, Table) -> +get_function(ducet, Table) -> % R1 is only for encoding to binary, not reassign. R1 = get_reassign_function(Table, 1), @@ -52,26 +52,26 @@ get_function(ducet, Table) -> MF = fun(Value) -> case ets:member(LTable, Value) of true -> true; - false -> + false -> case ets:member(MTable, Value) of true -> maybe; false -> false end end end, - + fun(member_function) -> MF; ({reassign_function, 1}) -> R1; % Return fun. ({reassign_function, 2}) -> R2; % Return fun. ({reassign_function, 3}) -> R3; ({reassign_function, 4}) -> R4; - ([_|_]=Value) -> + ([_|_]=Value) -> case F(Value) of - W when is_binary(W) -> + W when is_binary(W) -> bin_to_list2(W); - Other -> + Other -> Other - end + end end. get_reassign_function(Table, Lvl) -> @@ -94,7 +94,7 @@ get_val(Table, Val) -> do_after([{ducet, Table} | Tail]) -> do_after_ranges(Table), - + LTable = ets:new(ducet_lookup, [{write_concurrency, false}, {read_concurrency, true}]), do_after_lookup(Table, LTable), @@ -123,7 +123,7 @@ do_after_lookup(Table, LTable) -> do_after_lookup_next(Table, LTable, Index) end. % ets:safe_fixtable(Table, false), - + do_after_lookup_next(Table, LTable, PrevIndex) -> case ets:next(Table, PrevIndex) of '$end_of_table' -> @@ -154,7 +154,7 @@ do_after_ducet(Table, MTable) -> do_after_ducet_next(Table, MTable, Index) end. % ets:safe_fixtable(Table, false), - + do_after_ducet_next(Table, MTable, PrevIndex) -> case ets:next(Table, PrevIndex) of '$end_of_table' -> @@ -165,25 +165,25 @@ do_after_ducet_next(Table, MTable, PrevIndex) -> Index -> do_after_ducet_next(Table, MTable, Index) end. - + ducet_more(Table, MTable, In) -> - IF = fun(Val) -> + IF = fun(Val) -> % Insert new value ets:insert(Table, {Val, more}), Reversed = lists:reverse(Val), ets:insert(MTable, {Reversed}) end, - - LF = fun(Val) -> + + LF = fun(Val) -> % Lookup ets:member(Table, Val) end, - + do_ducet_more(LF, IF, lists:reverse(In)). -do_ducet_more(LF, IF, []) -> +do_ducet_more(_LF, _IF, []) -> ok; -do_ducet_more(LF, IF, [El]) -> +do_ducet_more(_LF, _IF, [_El]) -> ok; do_ducet_more(LF, IF, [_Last|ReversedBody] = _Codes) -> Body = lists:reverse(ReversedBody), @@ -192,7 +192,7 @@ do_ducet_more(LF, IF, [_Last|ReversedBody] = _Codes) -> false -> IF(Body) end, do_ducet_more(LF, IF, ReversedBody). - + @@ -206,14 +206,14 @@ do_after_ranges(Table) -> [{_, Val}] = ets:lookup(Table, Index), [Init|ValList] = bin_to_list(Val), ?DBG( - "~w:do_after_ranges: Init values: ~w. ~n", + "~w:do_after_ranges: Init values: ~w. ~n", [?MODULE, Init]), NewMax = lists:foldl(zip2fun(fun max/2), Init, ValList), NewMin = lists:foldl(zip2fun(fun min/2), Init, ValList), do_after_ranges_next(Table, Index, NewMax, NewMin) end. % ets:safe_fixtable(Table, false), - + do_after_ranges_next(Table, PrevIndex, Min, Max) -> case ets:next(Table, PrevIndex) of '$end_of_table' -> @@ -225,7 +225,7 @@ do_after_ranges_next(Table, PrevIndex, Min, Max) -> end, lists:foldl(InsFn(min), 1, Min), lists:foldl(InsFn(max), 1, Max), - + ok; Index -> [{_, Val}] = ets:lookup(Table, Index), @@ -234,7 +234,7 @@ do_after_ranges_next(Table, PrevIndex, Min, Max) -> NewMin = lists:foldl(zip2fun(fun min/2), Min, ValList), do_after_ranges_next(Table, Index, NewMin, NewMax) end. - + @@ -248,9 +248,9 @@ zip2fun(F) -> fun(L1, L2) -> lists:zipwith(F, L1, L2) end. max(V1, V2) when V1 > V2 -> V1; -max(V1, V2) -> V2. +max(_V1, V2) -> V2. min(V1, V2) when V1 < V2 -> V1; -min(V1, V2) -> V2. +min(_V1, V2) -> V2. %% bin_to_list(Bin) -> lists:map(fun([H|T]) -> T end, bin_to_list2(Bin)). bin_to_list(Bin) -> @@ -278,7 +278,7 @@ do_bin_to_list2(<<>>, Res) -> %% %% Parses "[.0000.0000.0000.0000]" to [<<0:8,0:16,0:16,0:16,0:16>>] -parse_el(El) -> +parse_el(El) -> ListOfInts = lists:reverse(parse_el(El, [], false, [])), el_to_bin(ListOfInts). @@ -304,27 +304,27 @@ el_res(Acc, Buf) when is_list(Acc), is_list(Buf) -> Hex = ux_unidata_parser:hex_to_int(lists:reverse(Acc)), [Hex|Buf]. -split_large_weights([Type, 0, 0, 0, 0], Res) -> Res; +split_large_weights([_Type, 0, 0, 0, 0], Res) -> Res; split_large_weights([Type, L1, L2, L3, L4], Res) -> L1Max = 16#FFDD, L2Max = 16#DD, L3Max = 16#DD, % 1F? L4Max = 16#FFDD, split_large_weights( - [Type, + [Type, if L1 0; true -> L1 - L1Max + 1 end, if L2 0; true -> L2 - L2Max + 1 end, if L3 0; true -> L3 - L3Max + 1 end, if L4 0; true -> L4 - L4Max + 1 end], - [[Type, + [[Type, if L1 L1; true -> L1Max end, if L2 L2; true -> L2Max end, if L3 L3; true -> L3Max end, if L4 L4; true -> L4Max end]|Res] ). - + el_to_bin(List) -> do_el_to_bin(List, <<>>). do_el_to_bin([[Type,L1,L2,L3,L4]|List], Bin) -> T = type_int(Type), @@ -350,7 +350,7 @@ parse_el_test_() -> F = fun parse_el/1, [?_assertEqual(F("[.0000.0000.0000.0000]"), <<>>) ,?_assertEqual(F("[.0001.0002.0003.0004]"), <<0:8, 1:16, 2:8, 3:8, 4:16>>) - ,?_assertEqual(F("[.0001.0002.0003.0004][*0005.0006.0007.0008]"), + ,?_assertEqual(F("[.0001.0002.0003.0004][*0005.0006.0007.0008]"), <<0:8, 1:16, 2:8, 3:8, 4:16, 1:8, 5:16, 6:8, 7:8, 8:16>>) ]. diff --git a/src/unidata/ux_unidata_server.erl b/src/unidata/ux_unidata_server.erl index a01e83f..3b3f045 100644 --- a/src/unidata/ux_unidata_server.erl +++ b/src/unidata/ux_unidata_server.erl @@ -1,12 +1,12 @@ %%% @doc This module provides the access to the store of default files. %%% When client runs a function from the ux_unidata module: -%%% 1. Code from ux_unidata_filelist module check the process dict +%%% 1. Code from ux_unidata_filelist module check the process dict %%% and the application enviroments. If they are unefined, then %%% it call this server. %%% 2. If this server already loaded this data, it returns it, and %%% the client code put it to the process dictionary. %%% 3. If requested data is not loaded, then this server runs -%%% an other server (ux_unidata_store), which parsed a default +%%% an other server (ux_unidata_store), which parsed a default %%% UNIDATA file. %%% @end %%% @@ -15,11 +15,9 @@ -include("ux.hrl"). -export([start_link/0]). --export([init/1, terminate/2, - handle_call/3, handle_info/2, handle_cast/2]). --export([set_default/1, get_default/1]). - -behavior(gen_server). +-export([init/1, terminate/2, handle_call/3, handle_info/2, handle_cast/2, code_change/3]). +-export([set_default/1, get_default/1]). %% Exported Client Functions %% Operation & Maintenance API @@ -63,14 +61,14 @@ spawn_waiter_reply(ReplyVal) -> %% Runs from a client process. wait_respond(WaiterPid) -> WaiterPid ! {reply_to, self()}, - {ok, Result} = + {ok, Result} = receive {reply_result, Val} -> {ok, Val} - after 20000 -> + after 20000 -> {error, timeout} end, Result. - + check_key(Key) -> case erlang:get(Key) of Pid when is_pid(Pid) -> @@ -90,8 +88,8 @@ check_key(Key) -> % Ref stores Key. % Key stores Pid of a waiter or Fun. % FromPid is a pid of a waiter process. -handle_info({'DOWN', Ref, process, FromPid, _Reason}, LoopData) -> - ?DBG("~w: Delete Pid = ~w from the process dictionary. ~n", +handle_info({'DOWN', Ref, process, _FromPid, _Reason}, LoopData) -> + ?DBG("~w: Delete Pid = ~w from the process dictionary. ~n", [?MODULE, FromPid]), case erlang:get(Ref) of @@ -106,11 +104,11 @@ handle_cast({waiter_reply, Key}, LoopData) -> %% I am using PD as a proxy (it is bad). -handle_call({get_default, Key} = V, From, LoopData) -> +handle_call({get_default, Key}, _From, LoopData) -> case ux_unidata_filelist:get_source_from(process, Key) of - undefined -> - LoaderFn = fun() -> - load_default(Key) + undefined -> + LoaderFn = fun() -> + load_default(Key) end, {WaiterPid, Ref} = spawn_waiter(LoaderFn, Key), put(Key, WaiterPid), @@ -118,14 +116,14 @@ handle_call({get_default, Key} = V, From, LoopData) -> Reply = WaiterPid, {reply, Reply, LoopData}; - Fun when is_function(Fun) -> + Fun when is_function(Fun) -> case Fun('test') of true -> {reply, Fun, LoopData}; % Restart the "dead" process, reload the function false -> - LoaderFn = fun() -> + LoaderFn = fun() -> Fun('reload') end, {WaiterPid, Ref} = spawn_waiter(LoaderFn, Key), @@ -135,7 +133,7 @@ handle_call({get_default, Key} = V, From, LoopData) -> end; %% We are still waiting. - WaiterPid when is_pid(WaiterPid) -> + WaiterPid when is_pid(WaiterPid) -> {reply, WaiterPid, LoopData} end; @@ -143,6 +141,9 @@ handle_call({set_default, Key}, _From, LoopData) -> Reply = ux_unidata_filelist:set_source(process, Key), {reply, Reply, LoopData}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + %% %% API %% @@ -151,7 +152,7 @@ handle_call({set_default, Key}, _From, LoopData) -> get_default(Key) -> Reply = gen_server:call(?MODULE, {get_default, Key}, 60000), case Reply of - Fun when is_function(Fun) -> + Fun when is_function(Fun) -> put(Key, Fun), % Registrate in the dict of the local process. Fun; WaiterPid when is_pid(WaiterPid) -> @@ -164,15 +165,14 @@ set_default(Key) -> %% %% Private helpers -%% +%% -%% Load all "columns" from the file, because it will be faster +%% Load all "columns" from the file, because it will be faster %% (minimize file readings). %% %% This function is LONG. -load_default({Parser, Type} = _Key) -> +load_default({Parser, _Type} = _Key) -> FileName = ux_unidata:get_source_file(Parser), % Types = [Type], Types = all, ux_unidata_filelist:set_source(process, Parser, Types, FileName). - diff --git a/src/unidata/ux_unidata_store.erl b/src/unidata/ux_unidata_store.erl index 0a6773d..3969ab0 100644 --- a/src/unidata/ux_unidata_store.erl +++ b/src/unidata/ux_unidata_store.erl @@ -3,17 +3,17 @@ -include("ux.hrl"). -export([start_link/2]). --export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2]). +-behavior(gen_server). +-export([init/1, terminate/2, handle_call/3, handle_cast/2, handle_info/2, code_change/3]). -export([monitor_client_process/2, get_funs/2]). % First argument is a server pid. -export([check_types/2, table_list/1, remove_type/2]). -% This functions are not connected to process directly, +% This functions are not connected to process directly, % but called from different processes. -export([get_env/1]). --behavior(gen_server). -record(state, { clients = [] :: [pid()], ets = [] :: [{atom(), integer()}], @@ -35,10 +35,10 @@ start_link(File, ClientPid) -> ClientEnv = get_env(File), Arguments = [File, ClientPid, ClientEnv], Opts = [], - Ret = gen_server:start_link(?MODULE, Arguments, Opts). + _Ret = gen_server:start_link(?MODULE, Arguments, Opts). -init([{ParserType, Types, FileName} = File, ClientPid, ClientEnv]) -> +init([{_ParserType, _Types, _FileName} = File, ClientPid, ClientEnv]) -> % We in the process code: extract client data. set_env(File, ClientEnv), @@ -61,35 +61,35 @@ terminate(_Reason, _LoopData) -> ok. -handle_call({check_types, _Types}, _From, +handle_call({check_types, _Types}, _From, #state{remain=[]} = LoopData) -> - ?DBG("~w~w: All types were already generated. ~n", + ?DBG("~w~w: All types were already generated. ~n", [?MODULE, self()]), Reply = ok, {reply, Reply, LoopData}; % Generate remains. -handle_call({check_types, all}, From, +handle_call({check_types, all}, From, #state{remain=Types} = LoopData) -> handle_call({check_types, Types}, From, LoopData); -handle_call({check_types, Types}, _From, +handle_call({check_types, Types}, _From, #state{ - types=RegistredTypes, - parser=ParserType, + types=RegistredTypes, + parser=ParserType, filename=FileName, remain=RemTypes, funs=Funs, ets=Ets} = LoopData) -> NewRemTypes = RemTypes -- Types, - {Reply, NewLoopData} = + {Reply, NewLoopData} = case NewRemTypes == RemTypes of - true -> - ?DBG("~w~w: Types were already generated. ~n", + true -> + ?DBG("~w~w: Types were already generated. ~n", [?MODULE, self()]), {ok, LoopData}; false -> AddTypes = Types -- RegistredTypes, File = {ParserType, AddTypes, FileName}, - {ok, AddedEts, _RemTypesWithRegistred} = + {ok, AddedEts, _RemTypesWithRegistred} = ux_unidata_parser:run(File), AddedFuns = ux_unidata_parser:get_functions(ParserType, AddedEts), {ok, LoopData#state{ @@ -98,10 +98,10 @@ handle_call({check_types, Types}, _From, remain=NewRemTypes, funs=AddedFuns ++ Funs }} - end, + end, {reply, Reply, NewLoopData}; -handle_call({monitor_client_pid, ClientPid}, _From, +handle_call({monitor_client_pid, ClientPid}, _From, #state{clients=Clients} = LoopData) -> NewLoopData = case lists:member(ClientPid, Clients) of true -> LoopData; @@ -114,46 +114,46 @@ handle_call({monitor_client_pid, ClientPid}, _From, {reply, Reply, NewLoopData}; -handle_call({get_funs, all}, _From, +handle_call({get_funs, all}, _From, #state{funs=Funs} = LoopData) -> Reply = Funs, {reply, Reply, LoopData}; -handle_call({get_funs, Types}, _From, +handle_call({get_funs, Types}, _From, #state{funs=Funs} = LoopData) -> - ?DBG("~w~w: Try get the list of the functions: ~w. ~n", + ?DBG("~w~w: Try get the list of the functions: ~w. ~n", [?MODULE, self(), Funs]), Reply = get_elems(Types, Funs), {reply, Reply, LoopData}; -handle_call(table_list, _From, +handle_call(table_list, _From, #state{ets=Ets} = LoopData) -> Reply = {ok, Ets}, {reply, Reply, LoopData}. -handle_info({'DOWN', _Ref, process, FromPid, _Reason}, +handle_info({'DOWN', _Ref, process, FromPid, _Reason}, #state{clients=Clients} = LoopData) -> ?DBG("~w~w: Delete the process ~w from the process list: ~w. ", [?MODULE, self(), FromPid, Clients]), NewClients = Clients -- [FromPid], case NewClients of - [] -> % wait 15 second and stop server. + [] -> % wait 15 second and stop server. Timeout = 15000, ?DBG("~w~w: Nobody use this server and ETS table. " - "Wait ~w ms and stop. ~n", + "Wait ~w ms and stop. ~n", [?MODULE, self(), Timeout]), - + timer:send_after(Timeout, self(), delete_timeout), ok; _ -> ok end, {noreply, LoopData#state{clients=NewClients}}; handle_info(delete_timeout, State=#state{clients=[]}) -> - ?DBG("~w~w: Nobody use this server and ETS table. Stop. ~n", + ?DBG("~w~w: Nobody use this server and ETS table. Stop. ~n", [?MODULE, self()]), {stop, normal, State}; % We have new clients. handle_info(delete_timeout, LoopData) -> - ?DBG("~w~w: New users use this server. Cancel stop. ~n", + ?DBG("~w~w: New users use this server. Cancel stop. ~n", [?MODULE, self()]), {noreply, LoopData}. @@ -163,7 +163,7 @@ handle_cast({run_parser, {ParserType, Types, FileName} = File}, % Run parser. {ok, Ets, RemTypes} = ux_unidata_parser:run(File), Funs = ux_unidata_parser:get_functions(ParserType, Ets), - ?DBG("~w~w: Init. Parser ~w generated ets: ~w and funs: ~w. ~n", + ?DBG("~w~w: Init. Parser ~w generated ets: ~w and funs: ~w. ~n", [?MODULE, self(), ParserType, Ets, Funs]), NewLoopData = LoopData#state{ ets = Ets, @@ -175,14 +175,14 @@ handle_cast({run_parser, {ParserType, Types, FileName} = File}, }, {noreply, NewLoopData}; -handle_cast({remove_type, Type}, +handle_cast({remove_type, Type}, #state{ets=Ets, types=Types, remain=Remain, funs=Funs} = LoopData) -> NewLoopData = case lists:keyfind(Type, 1, Ets) of false -> LoopData; - {Type, Table} -> + {Type, Table} -> true = ets:delete(Table), NewEts = lists:keydelete(Type, 1, Ets), NewFuns = lists:keydelete(Type, 1, Funs), @@ -195,6 +195,9 @@ handle_cast({remove_type, Type}, end, {noreply, NewLoopData}. +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + %% Monitor a proccess which called this function. %% ServerPid is a pid of gen_server with ETS. %% If all clients die then gen_server dies. @@ -229,10 +232,10 @@ do_get_elems([], _Elems, Acc) -> Acc. set_monitor(ClientPid) -> - ?DBG("~w~w: Set the monitor on the process ~w. ~n", + ?DBG("~w~w: Set the monitor on the process ~w. ~n", [?MODULE, self(), ClientPid]), erlang:monitor(process, ClientPid). - + diff --git a/src/unidata/ux_unidata_store_sup.erl b/src/unidata/ux_unidata_store_sup.erl index 6488ac6..e6a3c53 100644 --- a/src/unidata/ux_unidata_store_sup.erl +++ b/src/unidata/ux_unidata_store_sup.erl @@ -10,7 +10,7 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - ChildSpec = {ux_unidata_store, + ChildSpec = {ux_unidata_store, {ux_unidata_store, start_link, []}, temporary, 2000, worker, [ux_unidata_store]}, {ok, {{simple_one_for_one,0,1}, [ChildSpec]}}. @@ -20,10 +20,9 @@ init([]) -> read_file({_,_,_} = Filename, ClientPid) when is_pid(ClientPid) -> SupervisorName = ?MODULE, Ret = supervisor:start_child(SupervisorName, [Filename, ClientPid]), - {ok, ServerPid} = Ret. + {ok, _ServerPid} = Ret. + - %% @doc Restart this supervisor. restart() -> exit(whereis('ux_unidata_store_sup'), 'kill'). - diff --git a/src/utils/ux_opt_ranges.erl b/src/utils/ux_opt_ranges.erl index f9d8faa..ffc93ed 100644 --- a/src/utils/ux_opt_ranges.erl +++ b/src/utils/ux_opt_ranges.erl @@ -17,20 +17,20 @@ -export([in_list/1, search/2]). -in_list([H|_]=V) -> - SortedV = in_list_sort(V), +in_list([_H|_]=V) -> + %% SortedV = in_list_sort(V), R = erlang:list_to_tuple( - lists:map(fun(X) -> [] end, + lists:map(fun(_X) -> [] end, lists:seq(1,651))), do_in_list(V, R). search(Def, V) -> - SortedV = search_sort(V), + %% SortedV = search_sort(V), R = erlang:list_to_tuple( - lists:map(fun(X) -> [] end, + lists:map(fun(_X) -> [] end, lists:seq(1,651))), do_search(Def, V, R). @@ -56,8 +56,8 @@ do_in_list([], R) -> MiniList = erlang:element(I, MR), ux_ranges:in_list(MiniList, X) end. - - + + % skip do_search(Def, [{_,Def}|T], R) -> @@ -88,18 +88,18 @@ do_search(Def, [], R) -> end end. - - -set_elem(H, V, R) + + +set_elem(H, V, R) when is_tuple(R) -> I = index(H), E = erlang:element(I, R), erlang:setelement(I, R, [V|E]). - -set_elem_i(I, V, R) + +set_elem_i(I, V, R) when is_tuple(R) -> E = erlang:element(I, R), erlang:setelement(I, R, [V|E]). @@ -113,46 +113,37 @@ fill_elem(I1, I2, V, R) when I1 fill_elem(NewI1, I2, V, NewR). -index(N) when N > 65000 -> +index(N) when N > 65000 -> 651; -index(N) -> +index(N) -> (N div 100) + 1. -in_list_sort(V) -> - MF = fun({From,To} = Key) -> - {From, Key}; - (From) -> - {From, From} - end, - - MF2 = fun({_,Key}) -> Key end, - - V1 = lists:map(MF, V), - V2 = lists:sort(V1), - lists:map(MF2, V2). - - - -search_sort(V) -> - MF = fun(Key) -> - case erlang:element(1, Key) of - ({From,To}) -> - {From, Key}; - (From) -> - {From, From} - end - end, - - MF2 = fun({_,Key}) -> Key end, - - V1 = lists:map(MF, V), - V2 = lists:sort(V1), - lists:map(MF2, V2). - - - - - - - +%% in_list_sort(V) -> +%% MF = fun({From,_To} = Key) -> +%% {From, Key}; +%% (From) -> +%% {From, From} +%% end, +%% +%% MF2 = fun({_,Key}) -> Key end, +%% +%% V1 = lists:map(MF, V), +%% V2 = lists:sort(V1), +%% lists:map(MF2, V2). + +%% search_sort(V) -> +%% MF = fun(Key) -> +%% case erlang:element(1, Key) of +%% ({From,_To}) -> +%% {From, Key}; +%% (From) -> +%% {From, From} +%% end +%% end, +%% +%% MF2 = fun({_,Key}) -> Key end, +%% +%% V1 = lists:map(MF, V), +%% V2 = lists:sort(V1), +%% lists:map(MF2, V2). diff --git a/src/utils/ux_ranges.erl b/src/utils/ux_ranges.erl index fcbacc7..0628802 100644 --- a/src/utils/ux_ranges.erl +++ b/src/utils/ux_ranges.erl @@ -16,28 +16,26 @@ -module(ux_ranges). -export([in_list/2,search/2]). --spec in_list([{integer(), integer()} | integer()], integer()) -> +-spec in_list([{integer(), integer()} | integer()], integer()) -> boolean(). -in_list([H|T], H) +in_list([H|_T], H) when is_integer(H) -> true; -in_list([{From, To}|T], V) +in_list([{From, To}|_T], V) when V >= From, V =< To -> true; -in_list([H|T], V) -> +in_list([_H|T], V) -> in_list(T, V); in_list([], _V) -> false. --spec search([{{integer(), integer()} | integer(), term()}], integer()) -> +-spec search([{{integer(), integer()} | integer(), term()}], integer()) -> boolean(). -search([{H,P}|T], H) +search([{H,P}|_T], H) when is_integer(H) -> P; -search([{{From, To},P}|T], V) +search([{{From, To},P}|_T], V) when V >= From, V =< To -> P; search([_H|T], V) -> search(T, V); search([], _V) -> false. - - diff --git a/src/ux_string.erl b/src/ux_string.erl index 52a1c9b..1f97eb0 100644 --- a/src/ux_string.erl +++ b/src/ux_string.erl @@ -23,7 +23,7 @@ %%% ===================================================================== %%% ===================================================================== -%%% Copyright 2011 Uvarov Michael +%%% Copyright 2011 Uvarov Michael %%% %%% Licensed under the Apache License, Version 2.0 (the "License"); %%% you may not use this file except in compliance with the License. @@ -53,20 +53,20 @@ split/2, split/3, to_lower/1, to_upper/1, to_string/1, - delete_types/2, delete_types/3, - filter_types/2, filter_types/3, + delete_types/2, delete_types/3, + filter_types/2, filter_types/3, explode_types/2, split_types/2, first_types/3, last_types/3, script/1, scripts/1, - freq/1, + freq/1, is_nfc/1, is_nfd/1, is_nfkc/1, is_nfkd/1, to_nfc/1, to_nfd/1, to_nfkc/1, to_nfkd/1, to_ncr/1, to_graphemes/1, reverse/1, - length/1, + length/1, first/2, last/2, extract_words/1, @@ -79,13 +79,13 @@ --define(ASSERT(TEST,TRUE,FALSE), case TEST of - true -> TRUE; +-define(ASSERT(TEST,TRUE,FALSE), case TEST of + true -> TRUE; false -> FALSE end). --define(ASSERT_IN_ARRAY_LAMBDA(TEST), case TEST of - true -> fun lists:member/2; +-define(ASSERT_IN_ARRAY_LAMBDA(TEST), case TEST of + true -> fun lists:member/2; false -> fun not_in_array/2 end). @@ -99,7 +99,7 @@ nfd_qc(V) -> ?UNIDATA:nfd_qc(V). nfkc_qc(V) -> ?UNIDATA:nfkc_qc(V). nfkd_qc(V) -> ?UNIDATA:nfkd_qc(V). is_compat(V) -> ?UNIDATA:is_compat(V). -comp(V1, V2) -> ?UNIDATA:comp(V1, V2). +%% comp(V1, V2) -> ?UNIDATA:comp(V1, V2). comp('skip_check') -> ?UNIDATA:comp('skip_check'). decomp(V) -> ?UNIDATA:decomp(V). @@ -154,27 +154,27 @@ not_in_array(X,Y) -> not lists:member(X,Y). %% ==String functions based on the UNIDATA== %% -%% @doc Returns various "character types" which can be used +%% @doc Returns various "character types" which can be used %% as a default categorization in implementations. %% Types: %% http://www.ksu.ru/eng/departments/ktk/test/perl/lib/unicode/UCDFF301.html#General%20Category %% @end %% ux_char:type(_) -> false. -types(Str) -> +types(Str) -> Fun = ux_char:type(skip_check), lists:map(Fun, Str). -%% @doc Returns a new string which is made from the chars of Str +%% @doc Returns a new string which is made from the chars of Str %% which are not a type from Types list. %% @end -spec delete_types([char_type()], string()) -> string() | none(). -delete_types(Types, Str) -> +delete_types(Types, Str) -> Fun = ux_char:type(skip_check), - lists:filter(fun(El) -> - not lists:member(Fun(El), Types) + lists:filter(fun(El) -> + not lists:member(Fun(El), Types) end, Str). %% @doc Stops delete_type/2 after Limit deleted chars. If Limit < 0, then @@ -183,21 +183,21 @@ delete_types(Types, Str) -> -spec delete_types([char_type()], string(), integer()) -> string() | none(). delete_types(Types, Str, Limit) when Limit > 0 -> - lists:reverse(get_types(Types, Str, Limit, [], true, + lists:reverse(get_types(Types, Str, Limit, [], true, fun not_in_array/2, 0, -1)); delete_types(Types, Str, Limit) when Limit < 0 -> - lists:reverse(get_types(Types, Str, Limit, [], true, + lists:reverse(get_types(Types, Str, Limit, [], true, fun not_in_array/2, 1, 0)). -%% @doc Returns a new string which is made from the chars of Str +%% @doc Returns a new string which is made from the chars of Str %% which are a type from Types list. % @end -spec filter_types([char_type()], string()) -> string() | none(). -filter_types(Types, Str) -> +filter_types(Types, Str) -> Fun = ux_char:type(skip_check), - lists:filter(fun(El) -> - lists:member(Fun(El), Types) + lists:filter(fun(El) -> + lists:member(Fun(El), Types) end, Str). %% @doc Stops after -Limit skipped chars. @@ -206,32 +206,32 @@ filter_types(Types, Str) -> filter_types(Types, Str, Limit) when Limit > 0 -> lists:reverse( - get_types(Types, Str, Limit, [], true, + get_types(Types, Str, Limit, [], true, fun lists:member/2, -1, 0)); filter_types(Types, Str, Limit) when Limit < 0 -> lists:reverse( - get_types(Types, Str, Limit, [], true, + get_types(Types, Str, Limit, [], true, fun lists:member/2, 0, 1)). %% @doc If Len<0, then gets first Len chars of type, which is in Types %% If Len>0, then gets first -Len chars of type, which is NOT in Types %% @end -spec first_types([char_type()], string(), integer()) -> string() | none(). -first_types(Types, Str, Len) -> +first_types(Types, Str, Len) -> lists:reverse( - get_types(Types, Str, Len, [], false, - ?ASSERT_IN_ARRAY_LAMBDA(Len>0), + get_types(Types, Str, Len, [], false, + ?ASSERT_IN_ARRAY_LAMBDA(Len>0), ?ASSERT(Len>0, -1, 1), 0)). %% @doc If Len<0, then gets last Len chars of type, which is in Types %% If Len>0, then gets last -Len chars of type, which is NOT in Types %% @end -spec last_types([char_type()], string(), integer()) -> string() | none(). -last_types(Types, Str, Len) -> - get_types(Types, lists:reverse(Str), Len, [], false, - ?ASSERT_IN_ARRAY_LAMBDA(Len>0), +last_types(Types, Str, Len) -> + get_types(Types, lists:reverse(Str), Len, [], false, + ?ASSERT_IN_ARRAY_LAMBDA(Len>0), ?ASSERT(Len>0, -1, 1), 0). - + %% @private %% @doc Return list of chars, for which Fun(CharType) return true. %% If Len = 0, then return a part of modified string concatinated with @@ -242,43 +242,43 @@ last_types(Types, Str, Len) -> %% @end get_types(_Types, [] = _Str, _ = _Len, Result, _, _, _, _) -> Result; get_types(_, _, 0, Result, false, _, _, _) -> Result; -get_types(_, Tail, 0, Result, true, _, _, _) -> +get_types(_, Tail, 0, Result, true, _, _, _) -> lists:reverse(Tail)++Result; -get_types(Types, [Char|Tail], +get_types(Types, [Char|Tail], Len, % Stop after Len chars Result, % Result array RetTail, % Concat tail with Result or not Fun, % Check function TrueStep, % Len+TrueStep, if Fun return true - FalseStep) -> + FalseStep) -> case Fun(ux_char:type(Char), Types) of - true -> get_types(Types, Tail, Len+TrueStep, [Char|Result], + true -> get_types(Types, Tail, Len+TrueStep, [Char|Result], RetTail, Fun, TrueStep, FalseStep); - false -> get_types(Types, Tail, Len+FalseStep, Result, + false -> get_types(Types, Tail, Len+FalseStep, Result, RetTail, Fun, TrueStep, FalseStep) end. -%% @doc Returns a new list of strings which are parts of Str splited +%% @doc Returns a new list of strings which are parts of Str splited %% by separator chars of a type from Types list. %% @end -spec explode_types([char_type()], string()) -> string(). -explode_types(Types, Str) -> +explode_types(Types, Str) -> explode_reverse(explode_types_cycle(Types, Str, [], [])). %% @private explode_types_cycle(_Types, [], [], Res) -> Res; explode_types_cycle(_Types, [], [_|_] = Buf, Res) -> [Buf|Res]; -explode_types_cycle(Types, [Char|Str], Buf, Res) -> +explode_types_cycle(Types, [Char|Str], Buf, Res) -> case lists:member(ux_char:type(Char), Types) of true -> explode_types_cycle(Types, Str, [], [Buf|Res]); false -> explode_types_cycle(Types, Str, [Char|Buf], Res) end. -%% @doc Returns a new list of strings which are parts of Str splited +%% @doc Returns a new list of strings which are parts of Str splited %% by separator chars of a type from Types list. Parts can not be %% empty. -%% @end +%% @end -spec split_types([char_type()], string()) -> string(). split_types(Types, Str) -> delete_empty(explode_types(Types, Str)). @@ -311,7 +311,7 @@ split(P1, P2, P3) -> delete_empty(explode(P1, P2, P3)). %% [{70,3},{68,1}]''' %% %% @end --spec freq(string()) -> dict(). +-spec freq(string()) -> dict(). freq(Str) -> do_freq(Str, dict:new()). @@ -358,17 +358,17 @@ do_freq([], Dict) -> Dict. (char(), string()) -> [string()]; (nonempty_string(), string()) -> [string()]. -explode([Delimeter], [_|_] = Str) when is_integer(Delimeter) -> +explode([Delimeter], [_|_] = Str) when is_integer(Delimeter) -> explode_simple(Delimeter, lists:reverse(Str), [], []); -explode(Delimeter, [_|_] = Str) when is_integer(Delimeter) -> +explode(Delimeter, [_|_] = Str) when is_integer(Delimeter) -> explode_simple(Delimeter, lists:reverse(Str), [], []); -explode([_|_] = Delimeter, [_|_] = Str) -> +explode([_|_] = Delimeter, [_|_] = Str) -> case explode_cycle(Delimeter, Str, [], []) of false -> [Str]; Res -> explode_reverse(Res) end; explode([_|_], []) -> []; -explode(Char, []) +explode(Char, []) when is_integer(Char) -> []. @@ -378,17 +378,17 @@ explode(Char, []) (nonempty_string(), string(), integer()) -> [string()]. explode(Delimeter, [_|_] = Str, Limit) when is_integer(Delimeter) -> - explode([Delimeter], [_|_] = Str, Limit); -explode([_|_] = Delimeter, [_|_] = Str, Limit) when Limit > 0 -> + explode([Delimeter], [_|_] = Str, Limit); +explode([_|_] = Delimeter, [_|_] = Str, Limit) when Limit > 0 -> explode_reverse(explode_cycle_pos(Delimeter, Str, [], [], Limit)); -explode([_|_] = Delimeter, [_|_] = Str, Limit) when Limit < 0 -> +explode([_|_] = Delimeter, [_|_] = Str, Limit) when Limit < 0 -> case explode_cycle(Delimeter, Str, [], []) of false -> []; Res -> explode_reverse(lists:nthtail(-Limit, Res)) end; explode([_|_] = Delimeter, [_|_] = Str, 0) -> explode(Delimeter, Str); explode([_|_], [], _) -> []; -explode(Char, [], _) +explode(Char, [], _) when is_integer(Char) -> []. @@ -396,7 +396,7 @@ explode(Char, [], _) %% @private -explode_reverse(Res) -> lists:map(fun lists:reverse/1, lists:reverse(Res)). +explode_reverse(Res) -> lists:map(fun lists:reverse/1, lists:reverse(Res)). %% @doc Simple and fast realization. %% Delimeter is one char. @@ -410,7 +410,7 @@ explode_simple(Delimeter, [H|T], Buf, Res) -> explode_simple(_ , [ ], [ ], Res) -> Res; explode_simple(_ , [ ], Buf, Res) -> [Buf | Res]. -%% @doc This function puts a part of the string before the delimeter in Buf, +%% @doc This function puts a part of the string before the delimeter in Buf, %% if the delimeter is a substring of Str, then return Buf. %% Buf is a reversed list of reversed parts of the string. %% Return false, if Delimeter is not a part of Str. @@ -420,7 +420,7 @@ explode_cycle(_, [], _, []) -> false; explode_cycle(_, [], Buf, Result) -> [Buf | Result]; explode_cycle(Delimeter, Str, Buf, Result) -> case explode_check(Delimeter, Str) of - false -> [C|Tail] = Str, + false -> [C|Tail] = Str, explode_cycle(Delimeter, Tail, [C|Buf], Result); Tail -> explode_cycle(Delimeter, Tail, [], [Buf | Result]) end. @@ -430,10 +430,10 @@ explode_cycle_pos(_, [], Buf, Result, _) -> [Buf|Result]; explode_cycle_pos(_, [_|_] = Str, _, Result, 1) -> [lists:reverse(Str)|Result]; explode_cycle_pos(Delimeter, [_|_] = Str, Buf, Result, Limit) -> case explode_check(Delimeter, Str) of - false -> [C|Tail] = Str, - explode_cycle_pos(Delimeter, Tail, [C|Buf], Result, + false -> [C|Tail] = Str, + explode_cycle_pos(Delimeter, Tail, [C|Buf], Result, Limit); - Tail -> explode_cycle_pos(Delimeter, Tail, [], [Buf|Result], + Tail -> explode_cycle_pos(Delimeter, Tail, [], [Buf|Result], Limit-1) end. @@ -445,22 +445,22 @@ explode_check([[_|_]|_]=Delimeters, Str) -> explode_check1(Delimeters, Str); explode_check([_|_]=Delimeter, Str) -> explode_check2(Delimeter, Str). - + %% Delimeter is a list of a string(). explode_check1([[_|_]=Delimeter|T], [_|_]=Str) -> case explode_check2(Delimeter, Str) of false -> explode_check1(T, Str); - Result -> Result + Result -> Result end; explode_check1([], [_|_]) -> false. --spec explode_check2(string(), string()) -> +-spec explode_check2(string(), string()) -> false | string(). %% Delimeter is a string(). -explode_check2([Head|DelTail], [Head|Tail]) +explode_check2([Head|DelTail], [Head|Tail]) when is_integer(Head) -> explode_check2(DelTail, Tail); % Full match. @@ -577,10 +577,10 @@ to_upper(Str) -> %% -spec is_nf(fun(), list(), integer(), atom(), fun()) -> yes | no | maybe. -is_nf(CCC, [Head|Tail], LastCC, Result, CheckFun) -> +is_nf(CCC, [Head|Tail], LastCC, Result, CheckFun) -> case CCC(Head) of CC when (LastCC > CC) and (CC =/= 0) -> no; - CC -> + CC -> case CheckFun(Head) of n -> no; m -> is_nf(CCC, Tail, CC, maybe, CheckFun); @@ -595,7 +595,7 @@ is_nf(_CCC, [], _, Result, _) -> Result. %% http://unicode.org/reports/tr15/#Detecting_Normalization_Forms -spec is_nfc(list()) -> yes | no | maybe. -is_nfc(Str) when is_list(Str) -> +is_nfc(Str) when is_list(Str) -> CCC = ccc('skip_check'), QC = nfc_qc('skip_check'), is_nf(CCC, Str, 0, yes, QC). @@ -603,7 +603,7 @@ is_nfc(Str) when is_list(Str) -> -spec is_nfd(list()) -> yes | no | maybe. -is_nfd(Str) when is_list(Str) -> +is_nfd(Str) when is_list(Str) -> CCC = ccc('skip_check'), QC = nfd_qc('skip_check'), is_nf(CCC, Str, 0, yes, QC). @@ -631,7 +631,7 @@ is_nfkd(Str) when is_list(Str) -> -spec to_nfc(list()) -> list(). to_nfc([]) -> []; -to_nfc(Str) -> +to_nfc(Str) -> case is_nfc(Str) of yes -> Str; _ -> get_composition(to_nfd(Str)) @@ -648,38 +648,38 @@ to_nfkc([_|_] = Str) -> get_composition( -spec to_nfd(list()) -> list(). to_nfd([]) -> []; -to_nfd([_|_] = Str) -> +to_nfd([_|_] = Str) -> normalize(get_recursive_decomposition(true, Str)). -spec to_nfkd(list()) -> list(). to_nfkd([]) -> []; -to_nfkd([_|_] = Str) -> +to_nfkd([_|_] = Str) -> normalize(get_recursive_decomposition(false, Str)). %% @doc internal_decompose(Str) -%% Canonical If true bit is on in this byte, then selects the recursive +%% Canonical If true bit is on in this byte, then selects the recursive %% canonical decomposition, otherwise selects %% the recursive compatibility and canonical decomposition. %% @end %% @private -spec get_recursive_decomposition(atom() | function(), list()) -> list(). -get_recursive_decomposition(true, Str) -> +get_recursive_decomposition(true, Str) -> Canonical = is_compat(skip_check), Decomp = decomp(skip_check), get_recursive_decomposition(Decomp, Canonical, Str, []); -get_recursive_decomposition(false, Str) -> +get_recursive_decomposition(false, Str) -> Canonical = fun(_X) -> false end, % always false Decomp = decomp(skip_check), get_recursive_decomposition(Decomp, Canonical, Str, []); -get_recursive_decomposition(Canonical, Str) - when is_function(Canonical) -> +get_recursive_decomposition(Canonical, Str) + when is_function(Canonical) -> Decomp = decomp(skip_check), get_recursive_decomposition(Decomp, Canonical, Str, []). @@ -689,8 +689,8 @@ get_recursive_decomposition(Canonical, Str) % Skip ASCII %% @private -spec get_recursive_decomposition(fun(), fun(), list(), list()) -> list(). -get_recursive_decomposition(Decomp, Canonical, [Char|Tail], Result) - when Char < 128 -> % Cannot be decomposed +get_recursive_decomposition(Decomp, Canonical, [Char|Tail], Result) + when Char < 128 -> % Cannot be decomposed get_recursive_decomposition(Decomp, Canonical, Tail, [Char|Result]); @@ -711,13 +711,13 @@ get_recursive_decomposition(Decomp, Canonical, [Char|Tail], Result) -> case Decomp(Char) of [] -> get_recursive_decomposition(Decomp, Canonical, Tail, [Char|Result]); - Dec -> + Dec -> case Canonical(Char) of % not is_compat = singleton - true -> - get_recursive_decomposition(Decomp, Canonical, Tail, + true -> + get_recursive_decomposition(Decomp, Canonical, Tail, [Char|Result]); - false -> - get_recursive_decomposition(Decomp, Canonical, Tail, + false -> + get_recursive_decomposition(Decomp, Canonical, Tail, get_recursive_decomposition(Decomp, Canonical, Dec, Result)) end @@ -732,25 +732,25 @@ get_recursive_decomposition(_, _, [], Result) -> Result. %% @doc Normalize NFD or NFKD. -normalize(Str) -> +normalize(Str) -> CCC = ccc('skip_check'), normalize1(CCC, Str, [], []). %% @private -normalize1(_CCC, [], [ ], Result) -> +normalize1(_CCC, [], [ ], Result) -> Result; -normalize1(CCC, [], [_|_]=Buf, Result) -> +normalize1(_CCC, [], [_|_]=Buf, Result) -> normalize2(lists:reverse(Buf), Result); normalize1(CCC, [Char|Tail], Buf, Result) -> Class = CCC(Char), if - (Class == 0) and (Buf == []) -> + (Class == 0) and (Buf == []) -> normalize1(CCC, Tail, [], [Char | Result]); - (Class == 0) -> - normalize1(CCC, Tail, [], + (Class == 0) -> + normalize1(CCC, Tail, [], [Char | normalize2(lists:reverse(Buf), Result)]); true -> normalize1(CCC, Tail, [{Class, Char} | Buf], Result) end. @@ -768,10 +768,10 @@ normalize2(Buf, Result) -> %% @doc Return char from Buf with max ccc. %% @private -normalize3([{CharClass, _} = Value | Tail], _, MaxClass) - when CharClass > MaxClass -> +normalize3([{CharClass, _} = Value | Tail], _, MaxClass) + when CharClass > MaxClass -> normalize3(Tail, Value, CharClass); -normalize3([_|Tail], Value, MaxClass) -> +normalize3([_|Tail], Value, MaxClass) -> normalize3(Tail, Value, MaxClass); normalize3([], Value, _) -> Value. @@ -786,12 +786,12 @@ normalize3([], Value, _) -> Value. %% @doc Internal Composition Function. %% @private -get_composition([Char|Tail]) -> +get_composition([Char|Tail]) -> CCC = ccc('skip_check'), COMP = comp('skip_check'), lists:reverse( - get_composition(CCC, COMP, Tail, Char, + get_composition(CCC, COMP, Tail, Char, ?COMP_CHAR_CLASS(Char), [], []) ). @@ -801,13 +801,13 @@ get_composition([Char|Tail]) -> %% 2. check to see if two current characters are LV and T %% @end %% @private -get_composition(CCC, COMP, [VChar |Tail], LChar, 0, [], Result) +get_composition(CCC, COMP, [VChar |Tail], LChar, 0, [], Result) when ?CHAR_IS_HANGUL_L(LChar) and ?CHAR_IS_HANGUL_V(VChar) -> LIndex = LChar - ?HANGUL_LBASE, VIndex = VChar - ?HANGUL_VBASE, - LVChar = ?HANGUL_SBASE + ?HANGUL_TCOUNT + LVChar = ?HANGUL_SBASE + ?HANGUL_TCOUNT * (LIndex * ?HANGUL_VCOUNT + VIndex), case Tail of @@ -817,22 +817,22 @@ get_composition(CCC, COMP, [VChar |Tail], LChar, 0, [], Result) Result3 = [LVTChar|Result], case Tail2 of [Char|Tail3] -> - get_composition(CCC, COMP, Tail3, Char, + get_composition(CCC, COMP, Tail3, Char, ?COMP_CHAR_CLASS(Char), [], Result3); [] -> Result3 end; [Char|Tail2] -> - get_composition(CCC, COMP, Tail2, Char, + get_composition(CCC, COMP, Tail2, Char, ?COMP_CHAR_CLASS(Char), [], [LVChar|Result]); [] -> [LVChar|Result] end; -get_composition(CCC, COMP, [Char | Tail], LChar, 0, [], Result) +get_composition(CCC, COMP, [Char | Tail], LChar, 0, [], Result) when ?CHAR_IS_HANGUL_L(LChar) -> - get_composition(CCC, COMP, Tail, Char, + get_composition(CCC, COMP, Tail, Char, ?COMP_CHAR_CLASS(Char), [], [LChar|Result]); - -get_composition(CCC, COMP, [Char|Tail], LastChar, _, Mods, Result) + +get_composition(CCC, COMP, [Char|Tail], LastChar, _, Mods, Result) when Char < 128 -> NewResult = comp_append([LastChar|Result], Mods), get_composition(CCC, COMP, Tail, Char, 0, [], NewResult); @@ -841,19 +841,19 @@ get_composition(CCC, COMP, [Char|Tail], LastChar, LastClass, Mods, Result) -> CharClass = ccc(Char), Comp = COMP(LastChar, Char), if - (Comp =/= false) + (Comp =/= false) and ((LastClass < CharClass) or (LastClass == 0)) -> - get_composition(CCC, COMP, + get_composition(CCC, COMP, Tail, Comp, LastClass, Mods, Result); - (CharClass == 0) -> + (CharClass == 0) -> NewResult = comp_append([LastChar|Result], Mods), - get_composition(CCC, COMP, + get_composition(CCC, COMP, Tail, Char, CharClass, [], NewResult); - true -> + true -> NewMods = [Char|Mods], - get_composition(CCC, COMP, + get_composition(CCC, COMP, Tail, LastChar, CharClass, NewMods, Result) end; @@ -890,15 +890,15 @@ to_ncr([ ], Res) -> Res. %% [UAX29: UNICODE TEXT SEGMENTATION] %% (http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries). %% -%% It is important to recognize that what the user thinks of as -%% a "character"—a basic unit of a writing system for a language—may -%% not be just a single Unicode code point. Instead, that basic unit -%% may be made up of multiple Unicode code points. -%% To avoid ambiguity with the computer use of the term character, -%% this is called a user-perceived character. -%% For example, “G” + acute-accent is a user-perceived character: -%% users think of it as a single character, yet is actually represented -%% by two Unicode code points. These user-perceived characters are +%% It is important to recognize that what the user thinks of as +%% a "character"—a basic unit of a writing system for a language—may +%% not be just a single Unicode code point. Instead, that basic unit +%% may be made up of multiple Unicode code points. +%% To avoid ambiguity with the computer use of the term character, +%% this is called a user-perceived character. +%% For example, “G” + acute-accent is a user-perceived character: +%% users think of it as a single character, yet is actually represented +%% by two Unicode code points. These user-perceived characters are %% approximated by what is called a grapheme cluster, which can be %% determined programmatically. %% @end @@ -909,12 +909,12 @@ to_graphemes_raw([_|_]=S) -> [H|T] = ux_gb:split('extended', S), Buf = [H], Res = [], - + to_graphemes_raw(T, Buf, Res); to_graphemes_raw([]) -> []. - + %% @doc Returns not reversed result. %% @private -spec to_graphemes_raw(list(), string(), [string()]) -> @@ -930,18 +930,18 @@ to_graphemes_raw([], [_|_]=Buf, Res) -> [Buf|Res]; to_graphemes_raw([], []=_Buf, Res) -> Res. - - - + + + %% @doc Compute count of graphemes in the string. -length(S) -> +length(S) -> BS = ux_gb:split('extended', S), do_length(BS, 0). -do_length(['x',H|T], Len) -> +do_length(['x',_H|T], Len) -> do_length(T, Len); -do_length([H|T], Len) -> +do_length([_H|T], Len) -> do_length(T, Len + 1); do_length([], Len) -> Len. @@ -961,7 +961,7 @@ last(Str, Len) -> %% @doc Reverses the string graphemes. reverse(Str) -> reverse_flatten( - lists:reverse(to_graphemes_raw(Str)), + lists:reverse(to_graphemes_raw(Str)), [], []). %% [[1,2,3],[4,5,6]] => [6,5,4,3,2,1]. @@ -1023,13 +1023,13 @@ extract_words(S) -> script(S) -> F = ux_char:script('skip_check'), do_script(F, S, dict:new()). - + %% @private -do_script(F, [Char|Str], Dict) -> +do_script(F, [Char|Str], Dict) -> Script = F(Char), NewDict = dict:update_counter(Script, 1, Dict), do_script(F, Str, NewDict); -do_script(_F, [], Dict) -> +do_script(_F, [], Dict) -> L = dict:to_list(Dict), max(L). @@ -1043,7 +1043,7 @@ do_max([_|T], Max, S) -> do_max(T, Max, S); do_max([], _Max, S) -> S. - + @@ -1052,11 +1052,11 @@ do_max([], _Max, S) -> scripts(S) -> F = ux_char:script('skip_check'), do_scripts(F, S, sets:new()). - + %% @private -do_scripts(F, [H|T], Acc) -> +do_scripts(F, [H|T], Acc) -> Script = F(H), NewAcc = sets:add_element(Script, Acc), do_scripts(F, T, NewAcc); -do_scripts(_F, [], Dict) -> +do_scripts(_F, [], Dict) -> sets:to_list(Dict). diff --git a/src/ux_uca.erl b/src/ux_uca.erl index ee271c0..a9334e5 100644 --- a/src/ux_uca.erl +++ b/src/ux_uca.erl @@ -22,7 +22,7 @@ %%% ===================================================================== %%% ===================================================================== -%%% Copyright 2011 Uvarov Michael +%%% Copyright 2011 Uvarov Michael %%% %%% Licensed under the Apache License, Version 2.0 (the "License"); %%% you may not use this file except in compliance with the License. @@ -48,10 +48,10 @@ %%% Hangul Collation Requirements] %%% PS: There is the main source of information. %%% -%%% 2. [http://code.activestate.com/lists/perl-unicode/2163/ +%%% 2. [http://code.activestate.com/lists/perl-unicode/2163/ %%% Terminator weight for Hangul] %%% -%%% 3. [http://blogs.msdn.com/b/michkap/archive/2005/02/25/380266.aspx +%%% 3. [http://blogs.msdn.com/b/michkap/archive/2005/02/25/380266.aspx %%% Theory vs. practice for Korean text collation] %%% PS: there is no any practice. They do not the UCA :/ %%% @@ -72,7 +72,7 @@ %%% 10. [http://trapexit.org/String_Sorting_%28Natural%29 %%% String Sorting (Natural) in Erlang Cookbook] %%% -%%% +%%% %%% For hangul collation: %%% 11. [http://www.open-std.org/Jtc1/sc22/wg20/docs/n1037-Hangul%20Collation%20Requirements.htm %%% Hangul Collation Requirements] @@ -81,7 +81,7 @@ %%% 13. [http://en.wikipedia.org/wiki/KSX1001 KSX1001 on Wiki] %%% %%% -%%% == Levels == +%%% == Levels == %%% http://unicode.org/reports/tr10/#Multi_Level_Comparison %%% %%% * L1 Base characters @@ -96,7 +96,7 @@ %%% %%% %%% == Common configurations == -%%% +%%% %%% === Non-ignorable === %%% Variable collation elements are not reset to be ignorable, but %%% get the weights explicitly mentioned in the file. @@ -104,16 +104,16 @@ %%% * SPACE would have the value [.0209.0020.0002] %%% * Capital A would be unchanged, with the value [.06D9.0020.0008] %%% * Ignorables are unchanged. -%%% +%%% %%% Example: %%% ``` %%% C = ux_uca_options:get_options(non_ignorable). %%% ux_uca:sort_key(C, "Non-ignorable collation sort key"). ''' -%%% +%%% %%% %%% === Blanked === -%%% Variable collation elements and any subsequent ignorables -%%% are reset so that their weights at levels one through three are zero. +%%% Variable collation elements and any subsequent ignorables +%%% are reset so that their weights at levels one through three are zero. %%% For example, %%% %%% * SPACE would have the value [.0000.0000.0000] @@ -128,33 +128,33 @@ %%% ux_uca:sort_key(C, "Blanked collation sort key"). ''' %%% %%% -%%% === Shifted === +%%% === Shifted === %%% Variable collation elements are reset to zero at levels one through -%%% three. In addition, a new fourth-level weight is appended, whose value +%%% three. In addition, a new fourth-level weight is appended, whose value %%% depends on the type, as shown in Table 12. %%% Any subsequent primary or secondary ignorables following a variable are reset %%% so that their weights at levels one through four are zero. -%%% -%%% * A combining grave accent after a space would have the value +%%% +%%% * A combining grave accent after a space would have the value %%% [.0000.0000.0000.0000]. %%% * A combining grave accent after a Capital A would be unchanged. -%%% +%%% %%% Example: %%% ``` %%% C = ux_uca_options:get_options(shifted). %%% ux_uca:sort_key(C, "Shifted collation sort key"). ''' %%% %%% -%%% === Shift-trimmed === -%%% This option is the same as Shifted, except that all trailing -%%% FFFFs are trimmed from the sort key. +%%% === Shift-trimmed === +%%% This option is the same as Shifted, except that all trailing +%%% FFFFs are trimmed from the sort key. %%% This could be used to emulate POSIX behavior. %%% %%% Example: %%% ``` %%% C = ux_uca_options:get_options(shift_trimmed). %%% ux_uca:sort_key(C, "Shift-trimmed collation sort key"). ''' -%%% +%%% %%% %%% @end @@ -174,18 +174,18 @@ search/2, search/3, search/4 -% get_options/0, -% get_options/1, +% get_options/0, +% get_options/1, % get_options/2 ]). -import(ux_uca_utils, [ - do_alt/2, - do_alt/3, - do_extract/3, - get_ducet/0, - get_options/0, - split_levels/3, + do_alt/2, + do_alt/3, + do_extract/3, + get_ducet/0, + get_options/0, + split_levels/3, get_reassign_function/2]). @@ -212,7 +212,7 @@ | uncompressed % uncompressed list of weights . -% For hackers: +% For hackers: % In tr10 and ICU: % a weight is a sort key! % uca_weights is Collation Element (CE). @@ -250,7 +250,7 @@ -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). - + check_const_test_() -> Cases = fun(_) -> [?_assertEqual(l1("0"), [?COL_DECIMAL_START]) @@ -261,10 +261,10 @@ check_const_test_() -> ,?_assertEqual(l1([?HANGUL_TLAST]), [?COL_HANGUL_TLAST]) ] end, - {timeout, 60, + {timeout, 60, {setup, fun() -> l1("0") end, Cases}}. -l1(Str) -> +l1(Str) -> [L1 || [_, L1|_] <- ux_unidata:ducet(Str)]. -endif. @@ -292,16 +292,16 @@ do_compare(G1, G2) -> {stop, ____} -> lower; {____, stop} -> greater; - {{W1, NewG1}, {W2, NewG2}} + {{W1, NewG1}, {W2, NewG2}} when W1 =:= W2 -> do_compare(NewG1, NewG2); - {{W1, _NewG1}, {W2, _NewG2}} + {{W1, _NewG1}, {W2, _NewG2}} when W1 < W2 -> lower; - {{W1, _NewG1}, {W2, _NewG2}} + {{W1, _NewG1}, {W2, _NewG2}} when W1 > W2 -> greater end. - + %% @doc Convert the unicode string to the %% [http://unicode.org/reports/tr10/#Step_2 collation element array] %% @end @@ -325,17 +325,17 @@ do_sort_array(C, D, S, []=_W, A) -> do_sort_array(C, D, NewS, NewW, A); do_sort_array(C, D, S, [WH|WT], A) -> do_sort_array(C, D, S, WT, [WH|A]). - + %% @doc Convert the unicode string to the sort key. sort_key(S) -> C = get_options(), sort_key(C, S). - + %% @param C#sort_key_format{} %% @param S::string() sort_key(C=#uca_options{ - sort_key_format='binary', + sort_key_format='binary', case_sensitive=true}, S) -> ux_uca_sort_key_binary_cs:sort_key(C, preprocess(S)); sort_key(C=#uca_options{sort_key_format=F}, S) -> @@ -348,7 +348,7 @@ sort_key(C=#uca_options{sort_key_format=F}, S) -> ux_uca_sort_key_uncompressed:sort_key(C, preprocess(S)) end. - + -spec sort([string()]) -> [string()]. %% @doc Sort a list of strings. sort(Strings) -> @@ -359,11 +359,11 @@ sort(Strings) -> -spec sort(#uca_options{}, [string()]) -> [string()]. %% @doc Sort a list of strings. sort(C=#uca_options{}, Strings) -> - + % Step 1: produce array of sort keys - F = fun(S) -> + F = fun(S) -> Key = sort_key(C, S), - {Key, S} + {Key, S} end, Keys = lists:map(F, Strings), @@ -374,11 +374,9 @@ sort(C=#uca_options{}, Strings) -> RetFn = fun({_Key, S}) -> S end, lists:map(RetFn, SortedKeys). - - -weights(S) -> - C = get_options(), - weights(C, S). +%% weights(S) -> +%% C = get_options(), +%% weights(C, S). weights(C=#uca_options{strength=S}, Str) -> List = sort_array(C, Str), @@ -387,7 +385,6 @@ weights(C=#uca_options{strength=S}, Str) -> A = ux_uca_alt:get_alternate_function(C, D), do_weights(A, S, List, []). - %% Apply the alternate function for the list of weights. do_weights(A, S, [H|T], Acc) -> {NewA, Ints} = do_alt(A, H, S), @@ -396,13 +393,13 @@ do_weights(A, S, [H|T], Acc) -> [] -> Acc end, do_weights(NewA, S, T, NewAcc); -do_weights(A, S, [], Acc) -> +do_weights(A, _S, [], Acc) -> NewAcc = lists:reverse(Acc), {A, NewAcc}. %% %% Generator -%% +%% -spec generator(#uca_options{}, string()) -> uca_generator(). %generator(#uca_options{}, []) -> stop; @@ -417,8 +414,8 @@ generator(C=#uca_options{}, S) -> %% @param C::uca_options{} Configuration %% @param S::string() String %% @param D::fun() Ducet_reversed function -%% @param A::fun() Alternate function -%% @param W::[int()] ListOfWeights +%% @param A::fun() Alternate function +%% @param W::[int()] ListOfWeights %% @param R::[[int()]] Remain weights @@ -449,15 +446,15 @@ generator(C=#uca_options{}, S) -> do_generator(#uca_options{}=C, S, D, A, [WH|WT], R) -> {NewA, Ints} = do_alt(A, WH), case Ints of - [L1] when is_integer(L1) -> - F = fun() -> + [L1] when is_integer(L1) -> + F = fun() -> do_generator(C, S, D, NewA, WT, R) end, {L1, F}; [0|IT] -> % try extract next do_generator(C, S, D, NewA, WT, [IT|R]); - [L1|IT] when is_integer(L1) -> - F = fun() -> + [L1|IT] when is_integer(L1) -> + F = fun() -> do_generator(C, S, D, NewA, WT, [IT|R]) end, {L1, F}; @@ -486,8 +483,8 @@ do_generator2(1, []=_W, _R) -> do_generator2(_S, []=_W, []=_R) -> stop; do_generator2(S, []=_W, R) -> - F = fun() -> - do_generator2(S-1, lists:reverse(R), []) + F = fun() -> + do_generator2(S-1, lists:reverse(R), []) end, {0, F}; do_generator2(S, [[0=_WH]|WT], R) -> @@ -511,7 +508,7 @@ do_generator3(_S, []=_W, []=_R) -> stop; do_generator3(S, []=_W, R) -> F = fun() -> % not reverse - do_generator2(S-1, R, []) + do_generator2(S-1, R, []) end, {0, F}; do_generator3(S, [[0=_WH]|WT], R) -> @@ -525,7 +522,7 @@ do_generator3(S, [[WH|WR]|WT], R) -> F = fun() -> do_generator3(S, WT, [WR|R]) end, {WH, F}. - + %% 1|x|x => 1|x|x %% 1|1|x => 1|x|x %% 1|0|1 => 1|x|x @@ -535,26 +532,26 @@ prefix_weight([N,0|T]) -> NewN = N + 1, NewEl = [NewN|T], prefix_weight(NewEl); -prefix_weight([N]) -> +prefix_weight([_N]) -> false; -prefix_weight(El) -> +prefix_weight(El) -> El. prefix_weights([H|T], Acc) -> case prefix_weight([1|H]) of - false -> + false -> %skip: prefix_weights(T, Acc); - NewH -> + NewH -> NewAcc = [NewH|Acc], prefix_weights(T, NewAcc) end; prefix_weights([], Acc) -> lists:reverse(Acc). - + %% http://unicode.org/reports/tr10/#Searching --spec search(Target::string(), Pattern::string()) -> +-spec search(Target::string(), Pattern::string()) -> search_result(). search(T, P) -> @@ -564,9 +561,9 @@ search(T, P) -> -type search_result()::{string(),string(),string()}. --spec search(Target::string(), Pattern::string(), MatchStyle::atom()) -> +-spec search(Target::string(), Pattern::string(), MatchStyle::atom()) -> search_result(); - (#uca_options{}, Target::string(), Pattern::string()) -> + (#uca_options{}, Target::string(), Pattern::string()) -> search_result(). %% M is match-style: @@ -579,8 +576,8 @@ search(C=#uca_options{}, T, P) -> search(C, T, P, M). --spec search(#uca_options{}, Target::string(), Pattern::string(), - MatchStyle::atom()) -> +-spec search(#uca_options{}, Target::string(), Pattern::string(), + MatchStyle::atom()) -> search_result(). search(C=#uca_options{}, T, P, 'medium') -> @@ -590,23 +587,23 @@ search(C=#uca_options{}, T, P, 'medium') -> % Retrieve the sort key of the substring; {_NewAlt, AltW} = weights(NewC, P), % Convert to weights with prefix: - PW = prefix_weights(AltW, []), - + PW = prefix_weights(AltW, []), + D = get_ducet(), A = ux_uca_alt:get_alternate_function(C, D), Skipped = [], - + case do_search('first_minimal', C, P, D, A, PW, Skipped) of {[], _, []} -> % is equal do_search('first_minimal', C, T, D, A, PW, Skipped); - - {SubBefore, SubMatch, SubAfter} = SubV -> + + {SubBefore, _SubMatch, SubAfter} -> case do_search('maximal', C, T, D, A, PW, Skipped) of false -> false; - {MaxBefore, MaxMatch, MaxAfter} = MaxV -> + {MaxBefore, MaxMatch, MaxAfter} -> MinV = do_search('first_minimal', C, MaxMatch, D, A, PW, Skipped), {MinBefore, MinMatch, MinAfter} = MinV, @@ -616,25 +613,25 @@ search(C=#uca_options{}, T, P, 'medium') -> % "~w: " % "Max ~w. ~n" % "Min ~w. ~n" -% "Sub ~w. ~n", +% "Sub ~w. ~n", % [?MODULE, MaxV, MinV, SubV]), - - % concat the left part - {MedBeforeTail, MedMatch1} = + + % concat the left part + {MedBeforeTail, MedMatch1} = do_split(lists:reverse(MinBefore), lists:reverse(SubBefore), MinMatch), MedBefore = MaxBefore++lists:reverse(MedBeforeTail), - - % concat the right part - {MedAfterTail, MedMatch2} = + + % concat the right part + {MedAfterTail, MedMatch2} = do_split(MinAfter, SubAfter, lists:reverse(MedMatch1)), - + MedAfter = MedAfterTail++MaxAfter, - + MedMatch = lists:reverse(MedMatch2), {MedBefore, MedMatch, MedAfter} - end + end end; search(C, S, P, M) -> @@ -644,8 +641,8 @@ search(C, S, P, M) -> % Retrieve the sort key of the substring; {_NewAlt, AltW} = weights(NewC, P), % Convert to weights with prefix: - PW = prefix_weights(AltW, []), - + PW = prefix_weights(AltW, []), + D = get_ducet(), A = ux_uca_alt:get_alternate_function(C, D), Skipped = [], @@ -667,7 +664,7 @@ do_split(T1, _T2, Acc) -> do_search(M, C, [H|T]=S, D, A, PW, Skipped) -> case do_search_extract(M, C, S, D, A, PW) of 'stop' -> false; - false -> + false -> NewSkipped = [H|Skipped], do_search(M, C, T, D, A, PW, NewSkipped); {true, NewT} -> @@ -675,7 +672,7 @@ do_search(M, C, [H|T]=S, D, A, PW, Skipped) -> Matched = delete_tail(S, NewT), {NewSkipped, Matched, NewT} end; -do_search(M, _C, []=_S, _D, _A, _K, _Skipped) -> +do_search(_M, _C, []=_S, _D, _A, _K, _Skipped) -> false. %% Delete `length(Tail)' charactes from `From'. @@ -690,72 +687,68 @@ do_delete_tail(Rev, []) -> lists:reverse(Rev). --spec do_search_extract(M::atom(), #uca_options{}, S::string(), +-spec do_search_extract(M::atom(), #uca_options{}, S::string(), D::fun(), A::fun(), PW::uca_weights()) -> term(). -do_search_extract('first_minimal'=_M, +do_search_extract('first_minimal'=_M, C=#uca_options{strength=L}, S, D, A, PW) -> case do_extract(C, S, D) of - {[], _} -> + {[], _} -> 'stop'; - {[_|_]=NewW, NewS} -> + {[_|_]=NewW, NewS} -> {NewA, AltW} = do_weights(A, L, NewW, []), case is_ignorable_array(L, AltW) of true -> false; % reject ignorables - false -> - + false -> + case search_match(C, PW, AltW) of false -> false; true -> {true, NewS}; {'more', NewPW} -> NewM = 'minimal', - do_search_extract(NewM, - C, NewS, D, NewA, NewPW) + do_search_extract(NewM, + C, NewS, D, NewA, NewPW) end end end; -do_search_extract(M, +do_search_extract(M, C=#uca_options{strength=L}, S, D, A, PW) -> case do_extract(C, S, D) of - {[], _} -> + {[], _} -> 'stop'; - {[_|_]=NewW, NewS} -> + {[_|_]=NewW, NewS} -> {NewA, AltW} = do_weights(A, L, NewW, []), - + case search_match(C, PW, AltW) of - false -> + false -> false; - true when M=:='maximal' -> + true when M=:='maximal' -> NewNewS = delete_ignorables(C, NewS, D, A), {true, NewNewS}; - true -> + true -> {true, NewS}; {'more', NewPW} -> - do_search_extract(M, - C, NewS, D, NewA, NewPW) + do_search_extract(M, + C, NewS, D, NewA, NewPW) end end. -search_match(C=#uca_options{strength=MaxL}, PW, W) -> +search_match(#uca_options{strength=MaxL}, PW, W) -> L = 1, SkippedPW = [], % skipped weights from PW. SkippedW = [], % skipped weights from W. do_search_match(MaxL, L, PW, SkippedPW, W, SkippedW). - - - - %% @param MaxL strength %% @param L Level: 1->2...->MaxL %% @param PW Pattern weights (weights after `prefix_weights') @@ -766,17 +759,17 @@ search_match(C=#uca_options{strength=MaxL}, PW, W) -> % skip ignorable do_search_match(MaxL, L, PW, SkippedPW, [[0]=_HW|TW], SkippedW) -> do_search_match(MaxL, L, PW, SkippedPW, TW, SkippedW); - + % skip and save the tail do_search_match(MaxL, L, PW, SkippedPW, [[0|THW]=_HW|TW], SkippedW) -> NewSkippedW = [THW|SkippedW], do_search_match(MaxL, L, PW, SkippedPW, TW, NewSkippedW); % matched H -do_search_match(MaxL, L, [[L,H|THS]=_HS|TS], SkippedPW, +do_search_match(MaxL, L, [[L,H|THS]=_HS|TS], SkippedPW, [[H|HHW]=_HW|TW], SkippedW) -> NewHS = prefix_weight([L+1|THS]), - NewSkippedPW = + NewSkippedPW = case NewHS of false -> %skip: @@ -785,21 +778,21 @@ do_search_match(MaxL, L, [[L,H|THS]=_HS|TS], SkippedPW, [NewHS|SkippedPW] end, - NewSkippedW = + NewSkippedW = case HHW of [] -> SkippedW; _ -> [HHW|SkippedW] end, - do_search_match(MaxL, L, TS, NewSkippedPW, + do_search_match(MaxL, L, TS, NewSkippedPW, TW, NewSkippedW); - + % skip WTF do_search_match(MaxL, L, PW, SkippedPW, [[]=_HW|TW], SkippedW) -> do_search_match(MaxL, L, PW, SkippedPW, TW, SkippedW); - + % All levels was matched -do_search_match(MaxL, L, PW, SkippedPW, []=_W, []=_SkippedW) -> +do_search_match(_MaxL, _L, PW, SkippedPW, []=_W, []=_SkippedW) -> NewPW = append_skipped(PW, SkippedPW), case NewPW of [] -> true; @@ -818,8 +811,8 @@ do_search_match(MaxL, L, PW, SkippedPW, []=_W, SkippedW) -> do_search_match(_MaxL, _L, _PW, _SkippedPW, _W, _SkippedW) -> false. - - + + @@ -831,7 +824,7 @@ append_skipped(W, [H|T]) -> append_skipped(W, []) -> W. - + @@ -839,14 +832,14 @@ append_skipped(W, []) -> %% @private delete_ignorables(C=#uca_options{strength=L},[_|_]=S,D,A) -> {NewW,NewS} = do_extract(C, S, D), - {AltA, AltW} = do_weights(A, L, NewW, []), + {_AltA, AltW} = do_weights(A, L, NewW, []), case is_ignorable_array(L, AltW) of - true -> + true -> delete_ignorables(C,NewS,D,A); false -> S end; -delete_ignorables(_C,[]=_S,D,A) -> +delete_ignorables(_C,[]=_S,_D,_A) -> []. %% Warning: length(El) =< L @@ -858,11 +851,11 @@ is_ignorable_array(L, A) -> Mask = lists:duplicate(L, 0), lists:all(fun(El) -> El=:=Mask end, A). - + preprocess(S) -> %% TODO: normalization can be delayed. ux_string:to_nfd(S). - + %% %% Tests @@ -881,7 +874,7 @@ delete_tail_test_() -> split_levels_test_() -> F = fun(W) -> split_levels(1, false, W) end, F2 = fun(W) -> split_levels(2, true, W) end, - [{"Test common behavior.", + [{"Test common behavior.", [?_assertEqual(F([[1,2,3],[4,5,6],[7,8,9]]), {[1,4,7], [[2,3],[5,6],[8,9]]}) ,?_assertEqual(F([[1,2,3],[4],[7,8]]), {[1,4,7], [[2,3],[8]]}) ]} @@ -917,14 +910,14 @@ search_test_() -> ,{"http://unicode.org/reports/tr10/#Matches_Table", [{"The minimal match is the tightest one, because $! and %$ are " "ignored in the target.", - + [?TO(?_assertEqual(FF('minimal'), {"def$!","Abc","%$ghi"})) ,?TO(?_assertEqual(FF2('minimal'), {"def@!","Abc","%@ghi"})) ] } ,{"The medial one includes those characters that are binary equal.", - + [?TO(?_assertEqual(FF('medium'), {"def$","!Abc","%$ghi"})) ,?TO(?_assertEqual(FF2('medium'), {"def@","!Abc","%@ghi"})) ] @@ -933,7 +926,7 @@ search_test_() -> % TODO: Is this error in UCA? % ux_unidata:ducet("$"). % [[non_variable,5492,32,2,36]] - % + % % $ is not ignorable, but in example it is. ,{"The maximal match is the loosest one, including the surrounding" @@ -941,13 +934,13 @@ search_test_() -> % From example: % ?_assertEqual(FF('maximal'), {"def","$!Abc%$","ghi"}) - + % For real data: [?TO(?_assertEqual(FF('maximal'), {"def$","!Abc%","$ghi"})) ,?TO(?_assertEqual(FF2('maximal'), {"def","@!Abc%@","ghi"})) ] } ]}]. - + -endif. diff --git a/src/ux_unidata.erl b/src/ux_unidata.erl index 566e29b..1c97fdb 100644 --- a/src/ux_unidata.erl +++ b/src/ux_unidata.erl @@ -31,7 +31,7 @@ %%% ux_unidata_filelist contains the list of available files. %%% ux_unidata_filelist returns the anonymous function. %%% Fun extracts information from ETS table. -%%% The ETS tables were generated by ux_unidata_store. +%%% The ETS tables were generated by ux_unidata_store. %%% ux_unidata_store is the owner of the list of ETS tables. %%% ux_unidata_store runs ux_unidata_parser, which runs one of %%% ux_unidata_parser_*. @@ -48,8 +48,8 @@ -author('Uvarov Michael '). -export([get_source_file/1, get_test_file/1, open_test_file/1]). -export([char_to_upper/1, char_to_lower/1, is_upper/1, is_lower/1, - char_comment/1, char_type/1, ccc/1, - nfc_qc/1, nfd_qc/1, nfkc_qc/1, nfkd_qc/1, + char_comment/1, char_type/1, ccc/1, + nfc_qc/1, nfd_qc/1, nfkc_qc/1, nfkd_qc/1, is_comp_excl/1, is_compat/1, decomp/1, comp/2, comp/1, ducet/1, char_block/1, char_script/1, @@ -110,14 +110,14 @@ get_test_file('normalization_test') -> get_test_dir('ucd') ++ "NormalizationTest.txt.gz"; get_test_file('collation_test_shifted') -> - get_test_dir('uca') ++ "CollationTest/" + get_test_dir('uca') ++ "CollationTest/" % Slow, with comments. % "CollationTest_SHIFTED.txt"; "CollationTest_SHIFTED_SHORT.txt.gz"; get_test_file('collation_test_non_ignorable') -> - get_test_dir('uca') ++ "CollationTest/" -% "CollationTest_NON_IGNORABLE.txt", + get_test_dir('uca') ++ "CollationTest/" +% "CollationTest_NON_IGNORABLE.txt", % Fast version (data from slow version are equal). "CollationTest_NON_IGNORABLE_SHORT.txt.gz"; @@ -141,52 +141,52 @@ open_test_file(Id) -> --spec char_to_lower(char()) -> char(); +-spec char_to_lower(char()) -> char(); (skip_check) -> fun(). -char_to_lower(C) -> +char_to_lower(C) -> func(unidata, to_lower, C). --spec char_to_upper(char()) -> char(); +-spec char_to_upper(char()) -> char(); (skip_check) -> fun(). -char_to_upper(C) -> +char_to_upper(C) -> func(unidata, to_upper, C). --spec is_lower(char()) -> boolean(); +-spec is_lower(char()) -> boolean(); (skip_check) -> fun(). -is_lower(C) -> +is_lower(C) -> func(unidata, is_lower, C). --spec is_upper(char()) -> boolean(); +-spec is_upper(char()) -> boolean(); (skip_check) -> fun(). -is_upper(C) -> +is_upper(C) -> func(unidata, is_upper, C). -spec char_type(C::char()) -> atom(); (skip_check) -> fun(). -char_type(C) -> +char_type(C) -> func(unidata, type, C). -spec char_comment(C::char()) -> binary(); (skip_check) -> fun(). -char_comment(C) -> +char_comment(C) -> func(unidata, comment, C). -spec ccc(C::char()) -> ux_ccc(); (skip_check) -> fun(). -ccc(C) -> +ccc(C) -> func(unidata, ccc, C). @@ -194,35 +194,35 @@ ccc(C) -> -spec nfc_qc(C::char()) -> y | n | m; (skip_check) -> fun(). -nfc_qc(C) -> +nfc_qc(C) -> func(norm_props, nfc_qc, C). -spec nfd_qc(C::char()) -> y | n | m; (skip_check) -> fun(). -nfd_qc(C) -> +nfd_qc(C) -> func(norm_props, nfd_qc, C). -spec nfkc_qc(C::char()) -> y | n | m; (skip_check) -> fun(). -nfkc_qc(C) -> +nfkc_qc(C) -> func(norm_props, nfkc_qc, C). -spec nfkd_qc(C::char()) -> y | n | m; (skip_check) -> fun(). -nfkd_qc(C) -> +nfkd_qc(C) -> func(norm_props, nfkd_qc, C). -spec is_compat(C::char()) -> boolean(); (skip_check) -> fun(). -is_compat(C) -> +is_compat(C) -> func(unidata, is_compat, C). @@ -230,7 +230,7 @@ is_compat(C) -> -spec is_comp_excl(C::char()) -> boolean(); (skip_check) -> fun(). -is_comp_excl(C) -> +is_comp_excl(C) -> func(comp_exclusions, is_exclusion, C). @@ -242,35 +242,35 @@ ducet(L) -> func(allkeys, ducet, L). -spec comp(char(), char()) -> char() | false. -comp(C1, C2) -> +comp(C1, C2) -> func(unidata, comp, {C1, C2}). -comp('skip_check') -> +comp('skip_check') -> F = func(unidata, comp, 'skip_check'), fun(C1, C2) -> F({C1, C2}) end. - + -spec decomp(char()) -> list(); (skip_check) -> fun(). -decomp(C) -> +decomp(C) -> func(unidata, decomp, C). -spec char_block(C::char()) -> atom(); (skip_check) -> fun(). -char_block(C) -> +char_block(C) -> func(blocks, block, C). -spec char_script(C::char()) -> atom(); (skip_check) -> fun(). -char_script(C) -> +char_script(C) -> func(scripts, script, C). @@ -282,10 +282,10 @@ break_props('grapheme') -> break_props('word') -> Name = 'word_break_property', func(Name, Name, 'skip_check'). - -func(Parser, Type, Value) -> + +func(Parser, Type, Value) -> F = ux_unidata_filelist:get_source(Parser, Type), F(Value). @@ -295,14 +295,15 @@ w3(C) when 16#FF67 >= C, C >= 16#FF6F -> small_narrow_katakana; w3(C) when 16#FF71 >= C, C >= 16#FF9D -> narrow_katakana; w3(C) when 16#FFA0 >= C, C >= 16#FFDF -> narrow_hangul; w3(C) when 16#32D0 >= C, C >= 16#32FE -> circled_katakana; -w3(C) -> - case func(unidata, w3, C) of +w3(C) -> + case func(unidata, w3, C) of false -> case is_upper(C) of true -> upper; false -> false end; - - Type -> type end. + + _Type -> type + end. % Decomposition Type @@ -320,7 +321,7 @@ tertiary_weight(C) -> {compat, false} -> 16#04; {font, false} -> 16#05; {circle, false} -> 16#06; - + {false, upper} -> 16#08; {wide, upper} -> 16#09; {compat, upper} -> 16#0A; diff --git a/src/ux_wb.erl b/src/ux_wb.erl index 1c6f86c..e61a683 100644 --- a/src/ux_wb.erl +++ b/src/ux_wb.erl @@ -1,5 +1,5 @@ %%% @doc Default Word Cluster Boundary Breaker -%%% +%%% %%% [UTR29: Word Cluster Boundaries] %%% (http://unicode.org/reports/tr29/#Word_Cluster_Boundaries) %%% @end @@ -24,7 +24,7 @@ %% Adds the atom 'x' between non-breaked characters. -split(S) -> +split(S) -> Acc = [], % extract general classes @@ -37,14 +37,14 @@ split(S) -> {Types, expand(Res)}. - + words(S) -> {Types, Splitted} = ux_wb:split(S), Mod = false, Word = [], Acc = [], do_words(Mod, Splitted, Types, Word, Acc). - + %% Extract words. @@ -63,16 +63,16 @@ do_words(_Mod, ['-'|ST], TT, _Word, Acc) -> Mod = false, NewWord = [], do_words(Mod, ST, TT, NewWord, Acc); - + % Word. -do_words(_Mod, [SH|ST], [TH|TT], Word, Acc) +do_words(_Mod, [SH|ST], [TH|TT], Word, Acc) when TH=:='ALetter' -> Mod = true, NewWord = [SH|Word], do_words(Mod, ST, TT, NewWord, Acc); - + % Maybe word. -do_words(Mod, [SH|ST], [TH|TT], Word, Acc) -> +do_words(Mod, [SH|ST], [_TH|TT], Word, Acc) -> NewWord = [SH|Word], do_words(Mod, ST, TT, NewWord, Acc); @@ -84,11 +84,11 @@ do_words(true, [], [], [_|_] = Word, Acc) -> do_words(false, [], [], _Word, Acc) -> lists:reverse(Acc). - - - - - + + + + + % WB4 @@ -96,10 +96,10 @@ collapse(S, T) -> SR = [], TR = [], do_collapse(S, T, SR, TR). - -do_collapse([SH1,SH2|ST], [TH1,TH2|TT], SR, TR) - when (TH2=:='Extend' orelse + +do_collapse([SH1,SH2|ST], [TH1,TH2|TT], SR, TR) + when (TH2=:='Extend' orelse TH2=:='Format') andalso TH1=/='Newline' andalso TH1=/='CR' @@ -109,12 +109,12 @@ do_collapse([SH1,SH2|ST], [TH1,TH2|TT], SR, TR) do_collapse([SH|ST], [TH|TT], SR, TR) -> do_collapse(ST, TT, [SH|SR], [TH|TR]); - + do_collapse([], [], SR, TR) -> {lists:reverse(SR), lists:reverse(TR)}. -do_collapse2([SH|ST], [TH|TT], SR) +do_collapse2([SH|ST], [TH|TT], SR) when TH=:='Extend' ; TH=:='Format' -> do_collapse2(ST, TT, [SH|SR]); @@ -130,11 +130,11 @@ expand(S) -> do_expand([[_|_]=H|T], Acc) -> NewAcc = do_expand2(H, Acc), do_expand(T, NewAcc); - + do_expand([H|T], Acc) -> NewAcc = [H|Acc], do_expand(T, NewAcc); - + do_expand([], Acc) -> Acc. @@ -149,14 +149,14 @@ do_expand2([], Acc) -> % WB3 -do_split(_LT, [_CR,_LF|ST], +do_split(_LT, [_CR,_LF|ST], ['CR','LF'|TT], Acc) -> NewAcc = [?LF,?CR|Acc], do_split('LF', ST, TT, NewAcc); % WB3a -do_split(_LT, [SH|ST], - [TH1|TT = [_|_]], Acc) +do_split(_LT, [SH|ST], + [TH1|TT = [_|_]], Acc) when TH1=:='Newline' ; TH1=:='CR' ; TH1=:='LF' -> @@ -164,72 +164,72 @@ do_split(_LT, [SH|ST], do_split(TH1, ST, TT, NewAcc); % WB3b -do_split(_LT, [SH|ST], - [TH1|TT=[TH2|_]], Acc) +do_split(_LT, [SH|ST], + [TH1|TT=[TH2|_]], Acc) when TH2=:='Newline' ; TH2=:='CR' ; TH2=:='LF' -> NewAcc = ['-',SH|Acc], do_split(TH1, ST, TT, NewAcc); - + % WB5 -do_split(_LT, [SH|ST], +do_split(_LT, [SH|ST], ['ALetter'|TT = ['ALetter'|_]], Acc) -> NewAcc = [SH|Acc], do_split('ALetter', ST, TT, NewAcc); -% WB6 -do_split(_LT, [SH|ST], - ['ALetter'|TT = [TH2,'ALetter'|_]], Acc) +% WB6 +do_split(_LT, [SH|ST], + ['ALetter'|TT = [TH2,'ALetter'|_]], Acc) when TH2=:='MidLetter' ; TH2=:='MidNumLet' -> NewAcc = [SH|Acc], do_split('ALetter', ST, TT, NewAcc); - -% WB7 -do_split('ALetter', [SH|ST], - [TH1|TT = ['ALetter'|_]], Acc) + +% WB7 +do_split('ALetter', [SH|ST], + [TH1|TT = ['ALetter'|_]], Acc) when TH1=:='MidLetter' ; TH1=:='MidNumLet' -> NewAcc = [SH|Acc], do_split(TH1, ST, TT, NewAcc); - + % WB8 -do_split(_LT, [SH|ST], +do_split(_LT, [SH|ST], ['Numeric'|TT = ['Numeric'|_]], Acc) -> NewAcc = [SH|Acc], do_split('Numeric', ST, TT, NewAcc); % WB9 -do_split(_LT, [SH|ST], +do_split(_LT, [SH|ST], ['ALetter'|TT = ['Numeric'|_]], Acc) -> NewAcc = [SH|Acc], do_split('ALetter', ST, TT, NewAcc); % WB10 -do_split(_LT, [SH|ST], +do_split(_LT, [SH|ST], ['Numeric'|TT = ['ALetter'|_]], Acc) -> NewAcc = [SH|Acc], do_split('Numeric', ST, TT, NewAcc); - + % WB11 -do_split('Numeric', [SH|ST], - [TH1|TT = ['Numeric'|_]], Acc) +do_split('Numeric', [SH|ST], + [TH1|TT = ['Numeric'|_]], Acc) when TH1=:='MidNum' ; TH1=:='MidNumLet' -> NewAcc = [SH|Acc], do_split(TH1, ST, TT, NewAcc); % WB12 -do_split(_LT, [SH|ST], - ['Numeric'|TT = [TH2,'Numeric'|_]], Acc) +do_split(_LT, [SH|ST], + ['Numeric'|TT = [TH2,'Numeric'|_]], Acc) when TH2=:='MidNum' ; TH2=:='MidNumLet' -> NewAcc = [SH|Acc], @@ -237,14 +237,14 @@ do_split(_LT, [SH|ST], % WB13 -do_split(_LT, [SH|ST], +do_split(_LT, [SH|ST], ['Katakana'|TT = ['Katakana'|_]], Acc) -> NewAcc = [SH|Acc], do_split('Katakana', ST, TT, NewAcc); % WB13a -do_split(_LT, [SH|ST], - [TH1|TT = ['ExtendNumLet'|_]], Acc) +do_split(_LT, [SH|ST], + [TH1|TT = ['ExtendNumLet'|_]], Acc) when TH1=:='ALetter' ; TH1=:='Numeric' ; TH1=:='Katakana' @@ -253,8 +253,8 @@ do_split(_LT, [SH|ST], do_split(TH1, ST, TT, NewAcc); % WB13b -do_split(_LT, [SH|ST], - ['ExtendNumLet'|TT = [TH2|_]], Acc) +do_split(_LT, [SH|ST], + ['ExtendNumLet'|TT = [TH2|_]], Acc) when TH2=:='ALetter' ; TH2=:='Numeric' ; TH2=:='Katakana' -> @@ -264,16 +264,16 @@ do_split(_LT, [SH|ST], % Any -do_split(_LT, - [SH|ST], +do_split(_LT, + [SH|ST], [TH|TT=[_|_]], Acc) -> NewAcc = ['-',SH|Acc], do_split(TH, ST, TT, NewAcc); % Any -do_split(_LT, - [SH], - [TH], Acc) -> +do_split(_LT, + [SH], + [_TH], Acc) -> [SH|Acc]; do_split(_T, [], [], ['-'|Acc]) -> @@ -281,5 +281,3 @@ do_split(_T, [], [], ['-'|Acc]) -> do_split(_T, [], [], Acc) -> Acc. - -