Merge pull request #12668 from emqx/leap-year-bug-sql-fun

fix: port the changes for date_to_unix_ts SQL fun from 4.4

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -1181,7 +1181,7 @@ format_date(TimeUnit, Offset, FormatString, TimeEpoch) ->
 
 date_to_unix_ts(TimeUnit, FormatString, InputString) ->
     Unit = time_unit(TimeUnit),
-    emqx_utils_calendar:parse(InputString, Unit, FormatString).
+    emqx_utils_calendar:formatted_datetime_to_system_time(InputString, Unit, FormatString).
 
 date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) ->
     Unit = time_unit(TimeUnit),

apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl

@@ -1143,6 +1143,50 @@ timezone_to_offset_seconds_helper(FunctionName) ->
     apply_func(FunctionName, [local]),
     ok.
 
+t_date_to_unix_ts(_) ->
+    TestTab = [
+        {{"2024-03-01T10:30:38+08:00", second}, [
+            <<"second">>, <<"+08:00">>, <<"%Y-%m-%d %H-%M-%S">>, <<"2024-03-01 10:30:38">>
+        ]},
+        {{"2024-03-01T10:30:38.333+08:00", second}, [
+            <<"second">>, <<"+08:00">>, <<"%Y-%m-%d %H-%M-%S.%3N">>, <<"2024-03-01 10:30:38.333">>
+        ]},
+        {{"2024-03-01T10:30:38.333+08:00", millisecond}, [
+            <<"millisecond">>,
+            <<"+08:00">>,
+            <<"%Y-%m-%d %H-%M-%S.%3N">>,
+            <<"2024-03-01 10:30:38.333">>
+        ]},
+        {{"2024-03-01T10:30:38.333+08:00", microsecond}, [
+            <<"microsecond">>,
+            <<"+08:00">>,
+            <<"%Y-%m-%d %H-%M-%S.%3N">>,
+            <<"2024-03-01 10:30:38.333">>
+        ]},
+        {{"2024-03-01T10:30:38.333+08:00", nanosecond}, [
+            <<"nanosecond">>,
+            <<"+08:00">>,
+            <<"%Y-%m-%d %H-%M-%S.%3N">>,
+            <<"2024-03-01 10:30:38.333">>
+        ]},
+        {{"2024-03-01T10:30:38.333444+08:00", microsecond}, [
+            <<"microsecond">>,
+            <<"+08:00">>,
+            <<"%Y-%m-%d %H-%M-%S.%6N">>,
+            <<"2024-03-01 10:30:38.333444">>
+        ]}
+    ],
+    lists:foreach(
+        fun({{DateTime3339, Unit}, DateToTsArgs}) ->
+            ?assertEqual(
+                calendar:rfc3339_to_system_time(DateTime3339, [{unit, Unit}]),
+                apply_func(date_to_unix_ts, DateToTsArgs),
+                "Failed on test: " ++ DateTime3339 ++ "/" ++ atom_to_list(Unit)
+            )
+        end,
+        TestTab
+    ).
+
 t_parse_date_errors(_) ->
     ?assertError(
         bad_formatter_or_date,
@@ -1154,6 +1198,37 @@ t_parse_date_errors(_) ->
         bad_formatter_or_date,
         emqx_rule_funcs:date_to_unix_ts(second, <<"%y-%m-%d %H:%M:%S">>, <<"2022-05-26 10:40:12">>)
     ),
+    %% invalid formats
+    ?assertThrow(
+        {missing_date_part, month},
+        emqx_rule_funcs:date_to_unix_ts(
+            second, <<"%Y-%d %H:%M:%S">>, <<"2022-32 10:40:12">>
+        )
+    ),
+    ?assertThrow(
+        {missing_date_part, year},
+        emqx_rule_funcs:date_to_unix_ts(
+            second, <<"%H:%M:%S">>, <<"10:40:12">>
+        )
+    ),
+    ?assertError(
+        _,
+        emqx_rule_funcs:date_to_unix_ts(
+            second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-32 10:40:12">>
+        )
+    ),
+    ?assertError(
+        _,
+        emqx_rule_funcs:date_to_unix_ts(
+            second, <<"%Y-%m-%d %H:%M:%S">>, <<"2023-02-29 10:40:12">>
+        )
+    ),
+    ?assertError(
+        _,
+        emqx_rule_funcs:date_to_unix_ts(
+            second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-30 10:40:12">>
+        )
+    ),
 
     %% Compatibility test
     %% UTC+0
@@ -1173,25 +1248,42 @@ t_parse_date_errors(_) ->
         emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2022-05-26 10-40-12">>)
     ),
 
-    %% UTC+0
-    UnixTsLeap0 = 1582986700,
+    %% leap year checks
     ?assertEqual(
-        UnixTsLeap0,
-        emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2020-02-29 14:31:40">>)
+        %% UTC+0
+        1709217100,
+        emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-29 14:31:40">>)
     ),
-
-    %% UTC+0
-    UnixTsLeap1 = 1709297071,
     ?assertEqual(
-        UnixTsLeap1,
+        %% UTC+0
+        1709297071,
         emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">>)
     ),
-
-    %% UTC+0
-    UnixTsLeap2 = 1709535387,
     ?assertEqual(
-        UnixTsLeap2,
-        emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-04 06:56:27">>)
+        %% UTC+0
+        4107588271,
+        emqx_rule_funcs:date_to_unix_ts(second, <<"%Y-%m-%d %H:%M:%S">>, <<"2100-03-01 12:44:31">>)
+    ),
+    ?assertEqual(
+        %% UTC+8
+        1709188300,
+        emqx_rule_funcs:date_to_unix_ts(
+            second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-02-29 14:31:40">>
+        )
+    ),
+    ?assertEqual(
+        %% UTC+8
+        1709268271,
+        emqx_rule_funcs:date_to_unix_ts(
+            second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2024-03-01 12:44:31">>
+        )
+    ),
+    ?assertEqual(
+        %% UTC+8
+        4107559471,
+        emqx_rule_funcs:date_to_unix_ts(
+            second, <<"+08:00">>, <<"%Y-%m-%d %H:%M:%S">>, <<"2100-03-01 12:44:31">>
+        )
     ),
 
     %% None zero zone shift with millisecond level precision

apps/emqx_utils/src/emqx_utils_calendar.erl

@@ -22,7 +22,7 @@
     formatter/1,
     format/3,
     format/4,
-    parse/3,
+    formatted_datetime_to_system_time/3,
     offset_second/1
 ]).
 
@@ -48,8 +48,9 @@
 -define(DAYS_PER_YEAR, 365).
 -define(DAYS_PER_LEAP_YEAR, 366).
 -define(DAYS_FROM_0_TO_1970, 719528).
--define(SECONDS_FROM_0_TO_1970, (?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY)).
-
+-define(DAYS_FROM_0_TO_10000, 2932897).
+-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY).
+-define(SECONDS_FROM_0_TO_10000, (?DAYS_FROM_0_TO_10000 * ?SECONDS_PER_DAY)).
 %% the maximum value is the SECONDS_FROM_0_TO_10000 in the calendar.erl,
 %% here minus SECONDS_PER_DAY to tolerate timezone time offset,
 %% so the maximum date can reach 9999-12-31 which is ample.
@@ -171,10 +172,10 @@ format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) ->
 format(Time, Unit, Offset, Formatter) ->
     do_format(Time, time_unit(Unit), offset_second(Offset), Formatter).
 
-parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) ->
-    parse(DateStr, Unit, formatter(FormatterBin));
-parse(DateStr, Unit, Formatter) ->
-    do_parse(DateStr, Unit, Formatter).
+formatted_datetime_to_system_time(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) ->
+    formatted_datetime_to_system_time(DateStr, Unit, formatter(FormatterBin));
+formatted_datetime_to_system_time(DateStr, Unit, Formatter) ->
+    do_formatted_datetime_to_system_time(DateStr, Unit, Formatter).
 
 %%--------------------------------------------------------------------
 %% Time unit
@@ -467,56 +468,51 @@ padding(Data, _Len) ->
     Data.
 
 %%--------------------------------------------------------------------
-%% internal: parse part
+%% internal: formatted_datetime_to_system_time part
 %%--------------------------------------------------------------------
 
-do_parse(DateStr, Unit, Formatter) ->
+do_formatted_datetime_to_system_time(DateStr, Unit, Formatter) ->
     DateInfo = do_parse_date_str(DateStr, Formatter, #{}),
-    {Precise, PrecisionUnit} = precision(DateInfo),
-    Counter =
-        fun
-            (year, V, Res) ->
-                Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise);
-            (month, V, Res) ->
-                Dm = dym(maps:get(year, DateInfo, 0), V),
-                Res + Dm * ?SECONDS_PER_DAY * Precise;
-            (day, V, Res) ->
-                Res + (V * ?SECONDS_PER_DAY * Precise);
-            (hour, V, Res) ->
-                Res + (V * ?SECONDS_PER_HOUR * Precise);
-            (minute, V, Res) ->
-                Res + (V * ?SECONDS_PER_MINUTE * Precise);
-            (second, V, Res) ->
-                Res + V * Precise;
-            (millisecond, V, Res) ->
-                case PrecisionUnit of
-                    millisecond ->
-                        Res + V;
-                    microsecond ->
-                        Res + (V * 1000);
-                    nanosecond ->
-                        Res + (V * 1000000)
-                end;
-            (microsecond, V, Res) ->
-                case PrecisionUnit of
-                    microsecond ->
-                        Res + V;
-                    nanosecond ->
-                        Res + (V * 1000)
-                end;
-            (nanosecond, V, Res) ->
-                Res + V;
-            (parsed_offset, V, Res) ->
-                Res - V * Precise
-        end,
-    Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise),
-    erlang:convert_time_unit(Count, PrecisionUnit, Unit).
-
-precision(#{nanosecond := _}) -> {1000_000_000, nanosecond};
-precision(#{microsecond := _}) -> {1000_000, microsecond};
-precision(#{millisecond := _}) -> {1000, millisecond};
-precision(#{second := _}) -> {1, second};
-precision(_) -> {1, second}.
+    PrecisionUnit = precision(DateInfo),
+    ToPrecisionUnit = fun(Time, FromUnit) ->
+        erlang:convert_time_unit(Time, FromUnit, PrecisionUnit)
+    end,
+    GetRequiredPart = fun(Key) ->
+        case maps:get(Key, DateInfo, undefined) of
+            undefined -> throw({missing_date_part, Key});
+            Value -> Value
+        end
+    end,
+    GetOptionalPart = fun(Key) -> maps:get(Key, DateInfo, 0) end,
+    Year = GetRequiredPart(year),
+    Month = GetRequiredPart(month),
+    Day = GetRequiredPart(day),
+    Hour = GetRequiredPart(hour),
+    Min = GetRequiredPart(minute),
+    Sec = GetRequiredPart(second),
+    DateTime = {{Year, Month, Day}, {Hour, Min, Sec}},
+    TotalSecs = datetime_to_system_time(DateTime) - GetOptionalPart(parsed_offset),
+    check(TotalSecs, DateStr, Unit),
+    TotalTime =
+        ToPrecisionUnit(TotalSecs, second) +
+            ToPrecisionUnit(GetOptionalPart(millisecond), millisecond) +
+            ToPrecisionUnit(GetOptionalPart(microsecond), microsecond) +
+            ToPrecisionUnit(GetOptionalPart(nanosecond), nanosecond),
+    erlang:convert_time_unit(TotalTime, PrecisionUnit, Unit).
+
+check(Secs, _, _) when Secs >= -?SECONDS_FROM_0_TO_1970, Secs < ?SECONDS_FROM_0_TO_10000 ->
+    ok;
+check(_Secs, DateStr, Unit) ->
+    throw({bad_format, #{date_string => DateStr, to_unit => Unit}}).
+
+datetime_to_system_time(DateTime) ->
+    calendar:datetime_to_gregorian_seconds(DateTime) - ?SECONDS_FROM_0_TO_1970.
+
+precision(#{nanosecond := _}) -> nanosecond;
+precision(#{microsecond := _}) -> microsecond;
+precision(#{millisecond := _}) -> millisecond;
+precision(#{second := _}) -> second;
+precision(_) -> second.
 
 do_parse_date_str(<<>>, _, Result) ->
     Result;
@@ -564,27 +560,6 @@ date_size(timezone) -> 5;
 date_size(timezone1) -> 6;
 date_size(timezone2) -> 9.
 
-dym(Y, M) ->
-    case is_leap_year(Y) of
-        true when M > 2 ->
-            dm(M) + 1;
-        _ ->
-            dm(M)
-    end.
-
-dm(1) -> 0;
-dm(2) -> 31;
-dm(3) -> 59;
-dm(4) -> 90;
-dm(5) -> 120;
-dm(6) -> 151;
-dm(7) -> 181;
-dm(8) -> 212;
-dm(9) -> 243;
-dm(10) -> 273;
-dm(11) -> 304;
-dm(12) -> 334.
-
 str_to_int_or_error(Str, Error) ->
     case string:to_integer(Str) of
         {Int, []} ->

changes/ce/fix-12668.en.md

@@ -0,0 +1,2 @@
+Refactor the SQL function: `date_to_unix_ts()` by using `calendar:datetime_to_gregorian_seconds/1`.
+This change also added validation for the input date format.