diff --git a/apps/emqx/include/emqx_session.hrl b/apps/emqx/include/emqx_session.hrl index 3fea157ed..ebf20a9f1 100644 --- a/apps/emqx/include/emqx_session.hrl +++ b/apps/emqx/include/emqx_session.hrl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -17,39 +17,7 @@ -ifndef(EMQX_SESSION_HRL). -define(EMQX_SESSION_HRL, true). --record(session, { - %% Client's id - clientid :: emqx_types:clientid(), - id :: emqx_session:session_id(), - %% Is this session a persistent session i.e. was it started with Session-Expiry > 0 - is_persistent :: boolean(), - %% Client’s Subscriptions. - subscriptions :: map(), - %% Max subscriptions allowed - max_subscriptions :: non_neg_integer() | infinity, - %% Upgrade QoS? - upgrade_qos :: boolean(), - %% Client <- Broker: QoS1/2 messages sent to the client but - %% have not been unacked. - inflight :: emqx_inflight:inflight(), - %% All QoS1/2 messages published to when client is disconnected, - %% or QoS1/2 messages pending transmission to the Client. - %% - %% Optionally, QoS0 messages pending transmission to the Client. - mqueue :: emqx_mqueue:mqueue(), - %% Next packet id of the session - next_pkt_id = 1 :: emqx_types:packet_id(), - %% Retry interval for redelivering QoS1/2 messages (Unit: millisecond) - retry_interval :: timeout(), - %% Client -> Broker: QoS2 messages received from the client, but - %% have not been completely acknowledged - awaiting_rel :: map(), - %% Maximum number of awaiting QoS2 messages allowed - max_awaiting_rel :: non_neg_integer() | infinity, - %% Awaiting PUBREL Timeout (Unit: millisecond) - await_rel_timeout :: timeout(), - %% Created at - created_at :: pos_integer() -}). +-define(IS_SESSION_IMPL_MEM(S), (is_tuple(S) andalso element(1, S) =:= session)). +-define(IS_SESSION_IMPL_DS(S), (is_map_key(id, S))). -endif. diff --git a/apps/emqx/include/emqx_session_mem.hrl b/apps/emqx/include/emqx_session_mem.hrl new file mode 100644 index 000000000..9874a9018 --- /dev/null +++ b/apps/emqx/include/emqx_session_mem.hrl @@ -0,0 +1,55 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_SESSION_MEM_HRL). +-define(EMQX_SESSION_MEM_HRL, true). + +-record(session, { + %% Client's id + clientid :: emqx_types:clientid(), + id :: emqx_session:session_id(), + %% Is this session a persistent session i.e. was it started with Session-Expiry > 0 + is_persistent :: boolean(), + %% Client’s Subscriptions. + subscriptions :: map(), + %% Max subscriptions allowed + max_subscriptions :: non_neg_integer() | infinity, + %% Upgrade QoS? + upgrade_qos :: boolean(), + %% Client <- Broker: QoS1/2 messages sent to the client but + %% have not been unacked. + inflight :: emqx_inflight:inflight(), + %% All QoS1/2 messages published to when client is disconnected, + %% or QoS1/2 messages pending transmission to the Client. + %% + %% Optionally, QoS0 messages pending transmission to the Client. + mqueue :: emqx_mqueue:mqueue(), + %% Next packet id of the session + next_pkt_id = 1 :: emqx_types:packet_id(), + %% Retry interval for redelivering QoS1/2 messages (Unit: millisecond) + retry_interval :: timeout(), + %% Client -> Broker: QoS2 messages received from the client, but + %% have not been completely acknowledged + awaiting_rel :: map(), + %% Maximum number of awaiting QoS2 messages allowed + max_awaiting_rel :: non_neg_integer() | infinity, + %% Awaiting PUBREL Timeout (Unit: millisecond) + await_rel_timeout :: timeout(), + %% Created at + created_at :: pos_integer() +}). + +-endif. diff --git a/apps/emqx/integration_test/emqx_ds_SUITE.erl b/apps/emqx/integration_test/emqx_ds_SUITE.erl index fa30661e2..b042aa87a 100644 --- a/apps/emqx/integration_test/emqx_ds_SUITE.erl +++ b/apps/emqx/integration_test/emqx_ds_SUITE.erl @@ -72,30 +72,19 @@ end_per_testcase(_TestCase, _Config) -> %%------------------------------------------------------------------------------ cluster(#{n := N}) -> - Node1 = ds_SUITE1, - Spec = #{ - role => core, - join_to => emqx_cth_cluster:node_name(Node1), - apps => app_specs() - }, - [ - {Node1, Spec} - | lists:map( - fun(M) -> - Name = binary_to_atom(<<"ds_SUITE", (integer_to_binary(M))/binary>>), - {Name, Spec} - end, - lists:seq(2, N) - ) - ]. + Spec = #{role => core, apps => app_specs()}, + lists:map( + fun(M) -> + Name = list_to_atom("ds_SUITE" ++ integer_to_list(M)), + {Name, Spec} + end, + lists:seq(1, N) + ). app_specs() -> [ emqx_durable_storage, - {emqx, #{ - config => #{persistent_session_store => #{ds => true}}, - override_env => [{boot_modules, [broker, listeners]}] - }} + {emqx, "persistent_session_store = {ds = true}"} ]. get_mqtt_port(Node, Type) -> @@ -256,10 +245,9 @@ t_session_subscription_idempotency(Config) -> ?assertEqual([{ClientId, SubTopicFilterWords}], get_all_iterator_refs(Node1)), ?assertMatch({ok, [_]}, get_all_iterator_ids(Node1)), ?assertMatch( - {_IsNew = false, ClientId}, + {ok, #{}, #{SubTopicFilterWords := #{}}}, erpc:call(Node1, emqx_ds, session_open, [ClientId]) - ), - ok + ) end ), ok. diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 876fe66e0..12fa9625e 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -46,7 +46,6 @@ {emqx_node_rebalance_purge,1}. {emqx_node_rebalance_status,1}. {emqx_node_rebalance_status,2}. -{emqx_persistent_session,1}. {emqx_persistent_session_ds,1}. {emqx_plugins,1}. {emqx_prometheus,1}. diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 0f4987085..9954b514f 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -38,8 +38,7 @@ start(_Type, _Args) -> ok = maybe_load_config(), - ok = emqx_persistent_session:init_db_backend(), - _ = emqx_persistent_session_ds:init(), + _ = emqx_persistent_message:init(), ok = maybe_start_quicer(), ok = emqx_bpapi:start(), ok = emqx_alarm_handler:load(), diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index afa6dffe5..403e3757f 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -224,8 +224,17 @@ publish(Msg) when is_record(Msg, message) -> }), []; Msg1 = #message{topic = Topic} -> - _ = emqx_persistent_session_ds:persist_message(Msg1), - route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)) + PersistRes = persist_publish(Msg1), + PersistRes ++ route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)) + end. + +persist_publish(Msg) -> + case emqx_persistent_message:persist(Msg) of + ok -> + [persisted]; + {_SkipOrError, _Reason} -> + % TODO: log errors? + [] end. %% Called internally diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 0d83c60a6..8669aea8e 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -84,7 +84,7 @@ %% MQTT ClientInfo clientinfo :: emqx_types:clientinfo(), %% MQTT Session - session :: maybe(emqx_session:session()), + session :: maybe(emqx_session:t()), %% Keepalive keepalive :: maybe(emqx_keepalive:keepalive()), %% MQTT Will Msg @@ -104,7 +104,7 @@ %% Takeover takeover :: boolean(), %% Resume - resuming :: boolean(), + resuming :: false | _ReplayContext, %% Pending delivers when takeovering pendings :: list() }). @@ -130,14 +130,9 @@ -define(IS_MQTT_V5, #channel{conninfo = #{proto_ver := ?MQTT_PROTO_V5}}). --define(TIMER_TABLE, #{ - alive_timer => keepalive, - retry_timer => retry_delivery, - await_timer => expire_awaiting_rel, - expire_timer => expire_session, - will_timer => will_message, - quota_timer => expire_quota_limit -}). +-define(IS_COMMON_SESSION_TIMER(N), + ((N == retry_delivery) orelse (N == expire_awaiting_rel)) +). -define(LIMITER_ROUTING, message_routing). @@ -412,7 +407,7 @@ handle_in( #channel{clientinfo = ClientInfo, session = Session} ) -> case emqx_session:puback(ClientInfo, PacketId, Session) of - {ok, Msg, NSession} -> + {ok, Msg, [], NSession} -> ok = after_message_acked(ClientInfo, Msg, Properties), {ok, Channel#channel{session = NSession}}; {ok, Msg, Publishes, NSession} -> @@ -469,7 +464,7 @@ handle_in( } ) -> case emqx_session:pubcomp(ClientInfo, PacketId, Session) of - {ok, NSession} -> + {ok, [], NSession} -> {ok, Channel#channel{session = NSession}}; {ok, Publishes, NSession} -> handle_out(publish, Publishes, Channel#channel{session = NSession}); @@ -602,12 +597,10 @@ process_connect( {ok, #{session := Session, present := false}} -> NChannel = Channel#channel{session = Session}, handle_out(connack, {?RC_SUCCESS, sp(false), AckProps}, ensure_connected(NChannel)); - {ok, #{session := Session, present := true, pendings := Pendings}} -> - Pendings1 = lists:usort(lists:append(Pendings, emqx_utils:drain_deliver())), + {ok, #{session := Session, present := true, replay := ReplayContext}} -> NChannel = Channel#channel{ session = Session, - resuming = true, - pendings = Pendings1 + resuming = ReplayContext }, handle_out(connack, {?RC_SUCCESS, sp(true), AckProps}, ensure_connected(NChannel)); {error, client_id_unavailable} -> @@ -734,7 +727,7 @@ do_publish( {ok, PubRes, NSession} -> RC = pubrec_reason_code(PubRes), NChannel0 = Channel#channel{session = NSession}, - NChannel1 = ensure_timer(await_timer, NChannel0), + NChannel1 = ensure_timer(expire_awaiting_rel, NChannel0), NChannel2 = ensure_quota(PubRes, NChannel1), handle_out(pubrec, {PacketId, RC}, NChannel2); {error, RC = ?RC_PACKET_IDENTIFIER_IN_USE} -> @@ -765,7 +758,7 @@ ensure_quota(PubRes, Channel = #channel{quota = Limiter}) -> {ok, NLimiter} -> Channel#channel{quota = NLimiter}; {_, Intv, NLimiter} -> - ensure_timer(quota_timer, Intv, Channel#channel{quota = NLimiter}) + ensure_timer(expire_quota_limit, Intv, Channel#channel{quota = NLimiter}) end. -compile({inline, [pubrec_reason_code/1]}). @@ -909,8 +902,8 @@ maybe_update_expiry_interval( %% Check if the client turns off persistence (turning it on is disallowed) case EI =:= 0 andalso OldEI > 0 of true -> - NSession = emqx_session:unpersist(NChannel#channel.session), - NChannel#channel{session = NSession}; + ok = emqx_session:destroy(NChannel#channel.session), + NChannel#channel{session = undefined}; false -> NChannel end @@ -946,10 +939,12 @@ handle_deliver( clientinfo = ClientInfo } ) -> + %% NOTE + %% This is essentially part of `emqx_session_mem` logic, thus call it directly. Delivers1 = maybe_nack(Delivers), - NSession = emqx_session:enqueue(ClientInfo, Delivers1, Session), - NChannel = Channel#channel{session = NSession}, - {ok, NChannel}; + Messages = emqx_session:enrich_delivers(ClientInfo, Delivers1, Session), + NSession = emqx_session_mem:enqueue(ClientInfo, Messages, Session), + {ok, Channel#channel{session = NSession}}; handle_deliver( Delivers, Channel = #channel{ @@ -959,11 +954,11 @@ handle_deliver( } ) -> case emqx_session:deliver(ClientInfo, Delivers, Session) of + {ok, [], NSession} -> + {ok, Channel#channel{session = NSession}}; {ok, Publishes, NSession} -> NChannel = Channel#channel{session = NSession}, - handle_out(publish, Publishes, ensure_timer(retry_timer, NChannel)); - {ok, NSession} -> - {ok, Channel#channel{session = NSession}} + handle_out(publish, Publishes, ensure_timer(retry_delivery, NChannel)) end. %% Nack delivers from shared subscription @@ -1077,6 +1072,10 @@ return_connack(AckPacket, Channel) -> }, {Packets, NChannel2} = do_deliver(Publishes, NChannel1), Outgoing = [?REPLY_OUTGOING(Packets) || length(Packets) > 0], + %% NOTE + %% Session timers are not restored here, so there's a tiny chance that + %% the session becomes stuck, when it already has no place to track new + %% messages. {ok, Replies ++ Outgoing, NChannel2} end. @@ -1173,10 +1172,12 @@ handle_call( conninfo = #{clientid := ClientId} } ) -> - ok = emqx_session:takeover(Session), + %% NOTE + %% This is essentially part of `emqx_session_mem` logic, thus call it directly. + ok = emqx_session_mem:takeover(Session), %% TODO: Should not drain deliver here (side effect) Delivers = emqx_utils:drain_deliver(), - AllPendings = lists:append(Delivers, Pendings), + AllPendings = lists:append(Pendings, maybe_nack(Delivers)), ?tp( debug, emqx_channel_takeover_end, @@ -1199,7 +1200,7 @@ handle_call( SockInfo = maps:get(sockinfo, emqx_cm:get_chan_info(ClientId), #{}), ChanInfo1 = info(NChannel), emqx_cm:set_chan_info(ClientId, ChanInfo1#{sockinfo => SockInfo}), - reply(ok, reset_timer(alive_timer, NChannel)); + reply(ok, reset_timer(keepalive, NChannel)); handle_call(Req, Channel) -> ?SLOG(error, #{msg => "unexpected_call", call => Req}), reply(ignored, Channel). @@ -1231,14 +1232,18 @@ handle_info( {sock_closed, Reason}, Channel = #channel{ - conn_state = ConnState + conn_state = ConnState, + clientinfo = ClientInfo, + session = Session } ) when ConnState =:= connected orelse ConnState =:= reauthenticating -> + {Intent, Session1} = emqx_session:disconnect(ClientInfo, Session), Channel1 = ensure_disconnected(Reason, maybe_publish_will_msg(Channel)), - case maybe_shutdown(Reason, Channel1) of - {ok, Channel2} -> {ok, ?REPLY_EVENT(disconnected), Channel2}; + Channel2 = Channel1#channel{session = Session1}, + case maybe_shutdown(Reason, Intent, Channel2) of + {ok, Channel3} -> {ok, ?REPLY_EVENT(disconnected), Channel3}; Shutdown -> Shutdown end; handle_info({sock_closed, Reason}, Channel = #channel{conn_state = disconnected}) -> @@ -1305,66 +1310,54 @@ handle_timeout( case emqx_keepalive:check(StatVal, Keepalive) of {ok, NKeepalive} -> NChannel = Channel#channel{keepalive = NKeepalive}, - {ok, reset_timer(alive_timer, NChannel)}; + {ok, reset_timer(keepalive, NChannel)}; {error, timeout} -> handle_out(disconnect, ?RC_KEEP_ALIVE_TIMEOUT, Channel) end; handle_timeout( _TRef, - retry_delivery, + TimerName, Channel = #channel{conn_state = disconnected} -) -> +) when ?IS_COMMON_SESSION_TIMER(TimerName) -> {ok, Channel}; handle_timeout( _TRef, - retry_delivery, + TimerName, Channel = #channel{session = Session, clientinfo = ClientInfo} -) -> - case emqx_session:retry(ClientInfo, Session) of - {ok, NSession} -> +) when ?IS_COMMON_SESSION_TIMER(TimerName) -> + %% NOTE + %% Responsibility for these timers is smeared across both this module and the + %% `emqx_session` module: the latter holds configured timer intervals, and is + %% responsible for the actual timeout logic. Yet they are managed here, since + %% they are kind of common to all session implementations. + case emqx_session:handle_timeout(ClientInfo, TimerName, Session) of + {ok, Publishes, NSession} -> NChannel = Channel#channel{session = NSession}, - {ok, clean_timer(retry_timer, NChannel)}; + handle_out(publish, Publishes, clean_timer(TimerName, NChannel)); {ok, Publishes, Timeout, NSession} -> NChannel = Channel#channel{session = NSession}, - handle_out(publish, Publishes, reset_timer(retry_timer, Timeout, NChannel)) - end; -handle_timeout( - _TRef, - expire_awaiting_rel, - Channel = #channel{conn_state = disconnected} -) -> - {ok, Channel}; -handle_timeout( - _TRef, - expire_awaiting_rel, - Channel = #channel{session = Session, clientinfo = ClientInfo} -) -> - case emqx_session:expire(ClientInfo, awaiting_rel, Session) of - {ok, NSession} -> - NChannel = Channel#channel{session = NSession}, - {ok, clean_timer(await_timer, NChannel)}; - {ok, Timeout, NSession} -> - NChannel = Channel#channel{session = NSession}, - {ok, reset_timer(await_timer, Timeout, NChannel)} + handle_out(publish, Publishes, reset_timer(TimerName, Timeout, NChannel)) end; handle_timeout(_TRef, expire_session, Channel) -> shutdown(expired, Channel); handle_timeout( - _TRef, will_message, Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg} + _TRef, + will_message = TimerName, + Channel = #channel{clientinfo = ClientInfo, will_msg = WillMsg} ) -> (WillMsg =/= undefined) andalso publish_will_msg(ClientInfo, WillMsg), - {ok, clean_timer(will_timer, Channel#channel{will_msg = undefined})}; + {ok, clean_timer(TimerName, Channel#channel{will_msg = undefined})}; handle_timeout( _TRef, - expire_quota_limit, + expire_quota_limit = TimerName, #channel{quota = Quota} = Channel ) -> case emqx_limiter_container:retry(?LIMITER_ROUTING, Quota) of {_, Intv, Quota2} -> - Channel2 = ensure_timer(quota_timer, Intv, Channel#channel{quota = Quota2}), + Channel2 = ensure_timer(TimerName, Intv, Channel#channel{quota = Quota2}), {ok, Channel2}; {_, Quota2} -> - {ok, clean_timer(quota_timer, Channel#channel{quota = Quota2})} + {ok, clean_timer(TimerName, Channel#channel{quota = Quota2})} end; handle_timeout(TRef, Msg, Channel) -> case emqx_hooks:run_fold('client.timeout', [TRef, Msg], []) of @@ -1392,8 +1385,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> end. ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> - Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_utils:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Name), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> @@ -1405,15 +1397,15 @@ reset_timer(Name, Time, Channel) -> clean_timer(Name, Channel = #channel{timers = Timers}) -> Channel#channel{timers = maps:remove(Name, Timers)}. -interval(alive_timer, #channel{keepalive = KeepAlive}) -> +interval(keepalive, #channel{keepalive = KeepAlive}) -> emqx_keepalive:info(interval, KeepAlive); -interval(retry_timer, #channel{session = Session}) -> +interval(retry_delivery, #channel{session = Session}) -> emqx_session:info(retry_interval, Session); -interval(await_timer, #channel{session = Session}) -> +interval(expire_awaiting_rel, #channel{session = Session}) -> emqx_session:info(await_rel_timeout, Session); -interval(expire_timer, #channel{conninfo = ConnInfo}) -> +interval(expire_session, #channel{conninfo = ConnInfo}) -> maps:get(expiry_interval, ConnInfo); -interval(will_timer, #channel{will_msg = WillMsg}) -> +interval(will_message, #channel{will_msg = WillMsg}) -> timer:seconds(will_delay_interval(WillMsg)). %%-------------------------------------------------------------------- @@ -1783,7 +1775,7 @@ packing_alias(Packet, Channel) -> %% Check quota state check_quota_exceeded(_, #channel{timers = Timers}) -> - case maps:get(quota_timer, Timers, undefined) of + case maps:get(expire_quota_limit, Timers, undefined) of undefined -> ok; _ -> {error, ?RC_QUOTA_EXCEEDED} end. @@ -2044,15 +2036,15 @@ ensure_keepalive_timer(Interval, Channel = #channel{clientinfo = #{zone := Zone} Multiplier = get_mqtt_conf(Zone, keepalive_multiplier), RecvCnt = emqx_pd:get_counter(recv_pkt), Keepalive = emqx_keepalive:init(RecvCnt, round(timer:seconds(Interval) * Multiplier)), - ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). + ensure_timer(keepalive, Channel#channel{keepalive = Keepalive}). clear_keepalive(Channel = #channel{timers = Timers}) -> - case maps:get(alive_timer, Timers, undefined) of + case maps:get(keepalive, Timers, undefined) of undefined -> Channel; TRef -> emqx_utils:cancel_timer(TRef), - Channel#channel{timers = maps:without([alive_timer], Timers)} + Channel#channel{timers = maps:without([keepalive], Timers)} end. %%-------------------------------------------------------------------- %% Maybe Resume Session @@ -2061,30 +2053,25 @@ maybe_resume_session(#channel{resuming = false}) -> ignore; maybe_resume_session(#channel{ session = Session, - resuming = true, - pendings = Pendings, + resuming = ReplayContext, clientinfo = ClientInfo }) -> - {ok, Publishes, Session1} = emqx_session:replay(ClientInfo, Session), - case emqx_session:deliver(ClientInfo, Pendings, Session1) of - {ok, Session2} -> - {ok, Publishes, Session2}; - {ok, More, Session2} -> - {ok, lists:append(Publishes, More), Session2} - end. + emqx_session:replay(ClientInfo, ReplayContext, Session). %%-------------------------------------------------------------------- %% Maybe Shutdown the Channel -maybe_shutdown(Reason, Channel = #channel{conninfo = ConnInfo}) -> +maybe_shutdown(Reason, _Intent = idle, Channel = #channel{conninfo = ConnInfo}) -> case maps:get(expiry_interval, ConnInfo) of ?EXPIRE_INTERVAL_INFINITE -> {ok, Channel}; I when I > 0 -> - {ok, ensure_timer(expire_timer, I, Channel)}; + {ok, ensure_timer(expire_session, I, Channel)}; _ -> shutdown(Reason, Channel) - end. + end; +maybe_shutdown(Reason, _Intent = shutdown, Channel) -> + shutdown(Reason, Channel). %%-------------------------------------------------------------------- %% Parse Topic Filters @@ -2120,7 +2107,7 @@ maybe_publish_will_msg(Channel = #channel{clientinfo = ClientInfo, will_msg = Wi ok = publish_will_msg(ClientInfo, WillMsg), Channel#channel{will_msg = undefined}; I -> - ensure_timer(will_timer, timer:seconds(I), Channel) + ensure_timer(will_message, timer:seconds(I), Channel) end. will_delay_interval(WillMsg) -> diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index c795f7a33..cea22652d 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -19,9 +19,7 @@ -behaviour(gen_server). --include("emqx.hrl"). -include("emqx_cm.hrl"). --include("emqx_session.hrl"). -include("logger.hrl"). -include("types.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -48,14 +46,12 @@ set_chan_stats/2 ]). --export([get_chann_conn_mod/2]). - -export([ open_session/3, discard_session/1, discard_session/2, - takeover_session/1, - takeover_session/2, + takeover_session_begin/1, + takeover_session_end/1, kick_session/1, kick_session/2 ]). @@ -63,15 +59,14 @@ -export([ lookup_channels/1, lookup_channels/2, - - lookup_client/1 + lookup_client/1, + pick_channel/1 ]). %% Test/debug interface -export([ all_channels/0, - all_client_ids/0, - get_session_confs/2 + all_client_ids/0 ]). %% Client management @@ -96,12 +91,16 @@ clean_down/1, mark_channel_connected/1, mark_channel_disconnected/1, - get_connected_client_count/0, - takeover_finish/2, + get_connected_client_count/0 +]). +%% RPC targets +-export([ + takeover_session/2, + takeover_finish/2, do_kick_session/3, - do_get_chan_stats/2, do_get_chan_info/2, + do_get_chan_stats/2, do_get_chann_conn_mod/2 ]). @@ -118,6 +117,8 @@ _Stats :: emqx_types:stats() }. +-type takeover_state() :: {_ConnMod :: module(), _ChanPid :: pid()}. + -define(CHAN_STATS, [ {?CHAN_TAB, 'channels.count', 'channels.max'}, {?CHAN_TAB, 'sessions.count', 'sessions.max'}, @@ -259,98 +260,71 @@ set_chan_stats(ClientId, ChanPid, Stats) -> %% @doc Open a session. -spec open_session(boolean(), emqx_types:clientinfo(), emqx_types:conninfo()) -> {ok, #{ - session := emqx_session:session(), + session := emqx_session:t(), present := boolean(), - pendings => list() + replay => _ReplayContext }} | {error, Reason :: term()}. open_session(true, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Self = self(), - CleanStart = fun(_) -> + emqx_cm_locker:trans(ClientId, fun(_) -> ok = discard_session(ClientId), - ok = emqx_session:destroy(ClientId), + ok = emqx_session:destroy(ClientInfo, ConnInfo), create_register_session(ClientInfo, ConnInfo, Self) - end, - emqx_cm_locker:trans(ClientId, CleanStart); + end); open_session(false, ClientInfo = #{clientid := ClientId}, ConnInfo) -> Self = self(), - ResumeStart = fun(_) -> - case takeover_session(ClientId) of - {living, ConnMod, ChanPid, Session} -> - ok = emqx_session:resume(ClientInfo, Session), - case wrap_rpc(emqx_cm_proto_v2:takeover_finish(ConnMod, ChanPid)) of - {ok, Pendings} -> - clean_register_session(Session, Pendings, ClientInfo, ConnInfo, Self); - {error, _} -> - create_register_session(ClientInfo, ConnInfo, Self) - end; - none -> - create_register_session(ClientInfo, ConnInfo, Self) + emqx_cm_locker:trans(ClientId, fun(_) -> + case emqx_session:open(ClientInfo, ConnInfo) of + {true, Session, ReplayContext} -> + ok = register_channel(ClientId, Self, ConnInfo), + {ok, #{session => Session, present => true, replay => ReplayContext}}; + {false, Session} -> + ok = register_channel(ClientId, Self, ConnInfo), + {ok, #{session => Session, present => false}} end - end, - emqx_cm_locker:trans(ClientId, ResumeStart). - -create_session(ClientInfo, ConnInfo) -> - Options = get_session_confs(ClientInfo, ConnInfo), - Session = emqx_session:init_and_open(Options), - ok = emqx_metrics:inc('session.created'), - ok = emqx_hooks:run('session.created', [ClientInfo, emqx_session:info(Session)]), - Session. + end). create_register_session(ClientInfo = #{clientid := ClientId}, ConnInfo, ChanPid) -> - Session = create_session(ClientInfo, ConnInfo), + Session = emqx_session:create(ClientInfo, ConnInfo), ok = register_channel(ClientId, ChanPid, ConnInfo), {ok, #{session => Session, present => false}}. -clean_register_session(Session, Pendings, #{clientid := ClientId}, ConnInfo, ChanPid) -> - ok = register_channel(ClientId, ChanPid, ConnInfo), - {ok, #{ - session => clean_session(Session), - present => true, - pendings => clean_pendings(Pendings) - }}. +%% @doc Try to takeover a session from existing channel. +-spec takeover_session_begin(emqx_types:clientid()) -> + {ok, emqx_session_mem:session(), takeover_state()} | none. +takeover_session_begin(ClientId) -> + takeover_session_begin(ClientId, pick_channel(ClientId)). -get_session_confs(#{zone := Zone, clientid := ClientId}, #{ - receive_maximum := MaxInflight, expiry_interval := EI -}) -> - #{ - clientid => ClientId, - max_subscriptions => get_mqtt_conf(Zone, max_subscriptions), - upgrade_qos => get_mqtt_conf(Zone, upgrade_qos), - max_inflight => MaxInflight, - retry_interval => get_mqtt_conf(Zone, retry_interval), - await_rel_timeout => get_mqtt_conf(Zone, await_rel_timeout), - max_awaiting_rel => get_mqtt_conf(Zone, max_awaiting_rel), - mqueue => mqueue_confs(Zone), - %% TODO: Add conf for allowing/disallowing persistent sessions. - %% Note that the connection info is already enriched to have - %% default config values for session expiry. - is_persistent => EI > 0 - }. +takeover_session_begin(ClientId, ChanPid) when is_pid(ChanPid) -> + case takeover_session(ClientId, ChanPid) of + {living, ConnMod, Session} -> + {ok, Session, {ConnMod, ChanPid}}; + none -> + none + end; +takeover_session_begin(_ClientId, undefined) -> + none. -mqueue_confs(Zone) -> - #{ - max_len => get_mqtt_conf(Zone, max_mqueue_len), - store_qos0 => get_mqtt_conf(Zone, mqueue_store_qos0), - priorities => get_mqtt_conf(Zone, mqueue_priorities), - default_priority => get_mqtt_conf(Zone, mqueue_default_priority) - }. +%% @doc Conclude the session takeover process. +-spec takeover_session_end(takeover_state()) -> + {ok, _ReplayContext} | {error, _Reason}. +takeover_session_end({ConnMod, ChanPid}) -> + case wrap_rpc(emqx_cm_proto_v2:takeover_finish(ConnMod, ChanPid)) of + {ok, Pendings} -> + {ok, Pendings}; + {error, _} = Error -> + Error + end. -get_mqtt_conf(Zone, Key) -> - emqx_config:get_zone_conf(Zone, [mqtt, Key]). - -%% @doc Try to takeover a session. --spec takeover_session(emqx_types:clientid()) -> - none - | {living, atom(), pid(), emqx_session:session()} - | {persistent, emqx_session:session()} - | {expired, emqx_session:session()}. -takeover_session(ClientId) -> +-spec pick_channel(emqx_types:clientid()) -> + maybe(pid()). +pick_channel(ClientId) -> case lookup_channels(ClientId) of [] -> - emqx_session:lookup(ClientId); + undefined; [ChanPid] -> - takeover_session(ClientId, ChanPid); + ChanPid; ChanPids -> [ChanPid | StalePids] = lists:reverse(ChanPids), ?SLOG(warning, #{msg => "more_than_one_channel_found", chan_pids => ChanPids}), @@ -360,7 +334,7 @@ takeover_session(ClientId) -> end, StalePids ), - takeover_session(ClientId, ChanPid) + ChanPid end. takeover_finish(ConnMod, ChanPid) -> @@ -370,9 +344,10 @@ takeover_finish(ConnMod, ChanPid) -> ChanPid ). +%% @doc RPC Target @ emqx_cm_proto_v2:takeover_session/2 takeover_session(ClientId, Pid) -> try - do_takeover_session(ClientId, Pid) + do_takeover_begin(ClientId, Pid) catch _:R when R == noproc; @@ -380,25 +355,25 @@ takeover_session(ClientId, Pid) -> %% request_stepdown/3 R == unexpected_exception -> - emqx_session:lookup(ClientId); + none; % rpc_call/3 _:{'EXIT', {noproc, _}} -> - emqx_session:lookup(ClientId) + none end. -do_takeover_session(ClientId, ChanPid) when node(ChanPid) == node() -> - case get_chann_conn_mod(ClientId, ChanPid) of +do_takeover_begin(ClientId, ChanPid) when node(ChanPid) == node() -> + case do_get_chann_conn_mod(ClientId, ChanPid) of undefined -> - emqx_session:lookup(ClientId); + none; ConnMod when is_atom(ConnMod) -> case request_stepdown({takeover, 'begin'}, ConnMod, ChanPid) of {ok, Session} -> - {living, ConnMod, ChanPid, Session}; + {living, ConnMod, Session}; {error, Reason} -> error(Reason) end end; -do_takeover_session(ClientId, ChanPid) -> +do_takeover_begin(ClientId, ChanPid) -> wrap_rpc(emqx_cm_proto_v2:takeover_session(ClientId, ChanPid)). %% @doc Discard all the sessions identified by the ClientId. @@ -415,7 +390,7 @@ discard_session(ClientId) when is_binary(ClientId) -> %% benefits nobody. -spec request_stepdown(Action, module(), pid()) -> ok - | {ok, emqx_session:session() | list(emqx_types:deliver())} + | {ok, emqx_session:t() | _ReplayContext} | {error, term()} when Action :: kick | discard | {takeover, 'begin'} | {takeover, 'end'}. @@ -488,9 +463,10 @@ discard_session(ClientId, ChanPid) -> kick_session(ClientId, ChanPid) -> kick_session(kick, ClientId, ChanPid). +%% @doc RPC Target @ emqx_cm_proto_v2:kick_session/3 -spec do_kick_session(kick | discard, emqx_types:clientid(), chan_pid()) -> ok. -do_kick_session(Action, ClientId, ChanPid) -> - case get_chann_conn_mod(ClientId, ChanPid) of +do_kick_session(Action, ClientId, ChanPid) when node(ChanPid) =:= node() -> + case do_get_chann_conn_mod(ClientId, ChanPid) of undefined -> %% already deregistered ok; @@ -725,9 +701,6 @@ do_get_chann_conn_mod(ClientId, ChanPid) -> error:badarg -> undefined end. -get_chann_conn_mod(ClientId, ChanPid) -> - wrap_rpc(emqx_cm_proto_v2:get_chann_conn_mod(ClientId, ChanPid)). - mark_channel_connected(ChanPid) -> ?tp(emqx_cm_connected_client_count_inc, #{chan_pid => ChanPid}), ets:insert_new(?CHAN_LIVE_TAB, {ChanPid, true}), @@ -744,14 +717,3 @@ get_connected_client_count() -> undefined -> 0; Size -> Size end. - -clean_session(Session) -> - emqx_session:filter_queue(fun is_banned_msg/1, Session). - -clean_pendings(Pendings) -> - lists:filter(fun is_banned_msg/1, Pendings). - -is_banned_msg(#message{from = ClientId}) -> - [] =:= emqx_banned:look_up({clientid, ClientId}); -is_banned_msg({deliver, _Topic, Msg}) -> - is_banned_msg(Msg). diff --git a/apps/emqx/src/emqx_inflight.erl b/apps/emqx/src/emqx_inflight.erl index a3ff0ab10..96babd95a 100644 --- a/apps/emqx/src/emqx_inflight.erl +++ b/apps/emqx/src/emqx_inflight.erl @@ -28,6 +28,7 @@ update/3, resize/2, delete/2, + fold/3, values/1, to_list/1, to_list/2, @@ -77,6 +78,18 @@ delete(Key, ?INFLIGHT(MaxSize, Tree)) -> update(Key, Val, ?INFLIGHT(MaxSize, Tree)) -> ?INFLIGHT(MaxSize, gb_trees:update(Key, Val, Tree)). +-spec fold(fun((key(), Val :: term(), Acc) -> Acc), Acc, inflight()) -> Acc. +fold(FoldFun, AccIn, ?INFLIGHT(Tree)) -> + fold_iterator(FoldFun, AccIn, gb_trees:iterator(Tree)). + +fold_iterator(FoldFun, Acc, It) -> + case gb_trees:next(It) of + {Key, Val, ItNext} -> + fold_iterator(FoldFun, FoldFun(Key, Val, Acc), ItNext); + none -> + Acc + end. + -spec resize(integer(), inflight()) -> inflight(). resize(MaxSize, ?INFLIGHT(Tree)) -> ?INFLIGHT(MaxSize, Tree). diff --git a/apps/emqx/src/emqx_persistent_message.erl b/apps/emqx/src/emqx_persistent_message.erl new file mode 100644 index 000000000..7146332fc --- /dev/null +++ b/apps/emqx/src/emqx_persistent_message.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_persistent_message). + +-include("emqx.hrl"). + +-export([init/0]). +-export([is_store_enabled/0]). + +%% Message persistence +-export([ + persist/1, + serialize/1, + deserialize/1 +]). + +%% FIXME +-define(DS_SHARD_ID, <<"local">>). +-define(DEFAULT_KEYSPACE, default). +-define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). + +-define(WHEN_ENABLED(DO), + case is_store_enabled() of + true -> DO; + false -> {skipped, disabled} + end +). + +%%-------------------------------------------------------------------- + +init() -> + ?WHEN_ENABLED(begin + ok = emqx_ds:ensure_shard( + ?DS_SHARD, + #{ + dir => filename:join([ + emqx:data_dir(), + ds, + messages, + ?DEFAULT_KEYSPACE, + ?DS_SHARD_ID + ]) + } + ), + ok = emqx_persistent_session_ds_router:init_tables(), + ok + end). + +-spec is_store_enabled() -> boolean(). +is_store_enabled() -> + emqx_config:get([persistent_session_store, ds]). + +%%-------------------------------------------------------------------- + +-spec persist(emqx_types:message()) -> + ok | {skipped, _Reason} | {error, _TODO}. +persist(Msg) -> + ?WHEN_ENABLED( + case needs_persistence(Msg) andalso has_subscribers(Msg) of + true -> + store_message(Msg); + false -> + {skipped, needs_no_persistence} + end + ). + +needs_persistence(Msg) -> + not (emqx_message:get_flag(dup, Msg) orelse emqx_message:is_sys(Msg)). + +store_message(Msg) -> + ID = emqx_message:id(Msg), + Timestamp = emqx_guid:timestamp(ID), + Topic = emqx_topic:words(emqx_message:topic(Msg)), + emqx_ds_storage_layer:store(?DS_SHARD, ID, Timestamp, Topic, serialize(Msg)). + +has_subscribers(#message{topic = Topic}) -> + emqx_persistent_session_ds_router:has_any_route(Topic). + +%% + +serialize(Msg) -> + term_to_binary(emqx_message:to_map(Msg)). + +deserialize(Bin) -> + emqx_message:from_map(binary_to_term(Bin)). diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 6b25dd185..e56a05484 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -19,18 +19,40 @@ -include("emqx.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --export([init/0]). +-include("emqx_mqtt.hrl"). +%% Session API -export([ - persist_message/1, - open_session/1, - add_subscription/2, - del_subscription/2 + create/3, + open/2, + destroy/1 ]). -export([ - serialize_message/1, - deserialize_message/1 + info/2, + stats/1 +]). + +-export([ + subscribe/3, + unsubscribe/2, + get_subscription/2 +]). + +-export([ + publish/3, + puback/3, + pubrec/2, + pubrel/2, + pubcomp/3 +]). + +-export([ + deliver/3, + replay/3, + % handle_timeout/3, + disconnect/1, + terminate/2 ]). %% RPC @@ -49,157 +71,353 @@ -define(DEFAULT_KEYSPACE, default). -define(DS_SHARD, {?DEFAULT_KEYSPACE, ?DS_SHARD_ID}). --define(WHEN_ENABLED(DO), - case is_store_enabled() of - true -> DO; - false -> {skipped, disabled} - end -). +-type id() :: emqx_ds:session_id(). +-type iterator() :: emqx_ds:iterator(). +-type session() :: #{ + %% Client ID + id := id(), + %% When the session was created + created_at := timestamp(), + %% When the session should expire + expires_at := timestamp() | never, + %% Client’s Subscriptions. + iterators := #{topic() => iterator()}, + %% + props := map() +}. + +-type timestamp() :: emqx_utils_calendar:epoch_millisecond(). +-type topic() :: emqx_types:topic(). +-type clientinfo() :: emqx_types:clientinfo(). +-type conninfo() :: emqx_session:conninfo(). +-type replies() :: emqx_session:replies(). %% -init() -> - ?WHEN_ENABLED(begin - ok = emqx_ds:ensure_shard( - ?DS_SHARD, - #{ - dir => filename:join([ - emqx:data_dir(), - ds, - messages, - ?DEFAULT_KEYSPACE, - ?DS_SHARD_ID - ]) - } - ), - ok = emqx_persistent_session_ds_router:init_tables(), - ok - end). +-spec create(clientinfo(), conninfo(), emqx_session:conf()) -> + session(). +create(#{clientid := ClientID}, _ConnInfo, Conf) -> + % TODO: expiration + ensure_session(ClientID, Conf). -%% +-spec open(clientinfo(), conninfo()) -> + {_IsPresent :: true, session(), []} | false. +open(#{clientid := ClientID}, _ConnInfo) -> + %% NOTE + %% The fact that we need to concern about discarding all live channels here + %% is essentially a consequence of the in-memory session design, where we + %% have disconnected channels holding onto session state. Ideally, we should + %% somehow isolate those idling not-yet-expired sessions into a separate process + %% space, and move this call back into `emqx_cm` where it belongs. + ok = emqx_cm:discard_session(ClientID), + case open_session(ClientID) of + Session = #{} -> + {true, Session, []}; + false -> + false + end. --spec persist_message(emqx_types:message()) -> - ok | {skipped, _Reason} | {error, _TODO}. -persist_message(Msg) -> - ?WHEN_ENABLED( - case needs_persistence(Msg) andalso has_subscribers(Msg) of - true -> - store_message(Msg); - false -> - {skipped, needs_no_persistence} - end - ). - -needs_persistence(Msg) -> - not (emqx_message:get_flag(dup, Msg) orelse emqx_message:is_sys(Msg)). - -store_message(Msg) -> - ID = emqx_message:id(Msg), - Timestamp = emqx_guid:timestamp(ID), - Topic = emqx_topic:words(emqx_message:topic(Msg)), - emqx_ds_storage_layer:store( - ?DS_SHARD, ID, Timestamp, Topic, serialize_message(Msg) - ). - -has_subscribers(#message{topic = Topic}) -> - emqx_persistent_session_ds_router:has_any_route(Topic). +ensure_session(ClientID, Conf) -> + {ok, Session, #{}} = emqx_ds:session_ensure_new(ClientID, Conf), + Session#{iterators => #{}}. open_session(ClientID) -> - ?WHEN_ENABLED(emqx_ds:session_open(ClientID)). + case emqx_ds:session_open(ClientID) of + {ok, Session, Iterators} -> + Session#{iterators => prep_iterators(Iterators)}; + false -> + false + end. --spec add_subscription(emqx_types:topic(), emqx_ds:session_id()) -> - {ok, emqx_ds:iterator_id(), IsNew :: boolean()} | {skipped, disabled}. -add_subscription(TopicFilterBin, DSSessionID) -> - ?WHEN_ENABLED( - begin - %% N.B.: we chose to update the router before adding the subscription to the - %% session/iterator table. The reasoning for this is as follows: - %% - %% Messages matching this topic filter should start to be persisted as soon as - %% possible to avoid missing messages. If this is the first such persistent - %% session subscription, it's important to do so early on. - %% - %% This could, in turn, lead to some inconsistency: if such a route gets - %% created but the session/iterator data fails to be updated accordingly, we - %% have a dangling route. To remove such dangling routes, we may have a - %% periodic GC process that removes routes that do not have a matching - %% persistent subscription. Also, route operations use dirty mnesia - %% operations, which inherently have room for inconsistencies. - %% - %% In practice, we use the iterator reference table as a source of truth, - %% since it is guarded by a transaction context: we consider a subscription - %% operation to be successful if it ended up changing this table. Both router - %% and iterator information can be reconstructed from this table, if needed. - ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID), - TopicFilter = emqx_topic:words(TopicFilterBin), - {ok, IteratorID, StartMS, IsNew} = emqx_ds:session_add_iterator( - DSSessionID, TopicFilter - ), - Ctx = #{ - iterator_id => IteratorID, - start_time => StartMS, - is_new => IsNew - }, - ?tp(persistent_session_ds_iterator_added, Ctx), - ?tp_span( - persistent_session_ds_open_iterators, - Ctx, - ok = open_iterator_on_all_shards(TopicFilter, StartMS, IteratorID) - ), - {ok, IteratorID, IsNew} - end +prep_iterators(Iterators) -> + maps:fold( + fun(Topic, Iterator, Acc) -> Acc#{emqx_topic:join(Topic) => Iterator} end, + #{}, + Iterators ). --spec open_iterator_on_all_shards(emqx_topic:words(), emqx_ds:time(), emqx_ds:iterator_id()) -> ok. -open_iterator_on_all_shards(TopicFilter, StartMS, IteratorID) -> - ?tp(persistent_session_ds_will_open_iterators, #{ - iterator_id => IteratorID, - start_time => StartMS - }), +-spec destroy(session() | clientinfo()) -> ok. +destroy(#{id := ClientID}) -> + destroy_session(ClientID); +destroy(#{clientid := ClientID}) -> + destroy_session(ClientID). + +destroy_session(ClientID) -> + _ = ensure_all_iterators_closed(ClientID), + emqx_ds:session_drop(ClientID). + +%%-------------------------------------------------------------------- +%% Info, Stats +%%-------------------------------------------------------------------- + +info(Keys, Session) when is_list(Keys) -> + [{Key, info(Key, Session)} || Key <- Keys]; +info(id, #{id := ClientID}) -> + ClientID; +info(clientid, #{id := ClientID}) -> + ClientID; +info(created_at, #{created_at := CreatedAt}) -> + CreatedAt; +info(is_persistent, #{}) -> + true; +info(subscriptions, #{iterators := Iters}) -> + maps:map(fun(_, #{props := SubOpts}) -> SubOpts end, Iters); +info(subscriptions_cnt, #{iterators := Iters}) -> + maps:size(Iters); +info(subscriptions_max, #{props := Conf}) -> + maps:get(max_subscriptions, Conf); +info(upgrade_qos, #{props := Conf}) -> + maps:get(upgrade_qos, Conf); +% info(inflight, #sessmem{inflight = Inflight}) -> +% Inflight; +% info(inflight_cnt, #sessmem{inflight = Inflight}) -> +% emqx_inflight:size(Inflight); +% info(inflight_max, #sessmem{inflight = Inflight}) -> +% emqx_inflight:max_size(Inflight); +info(retry_interval, #{props := Conf}) -> + maps:get(retry_interval, Conf); +% info(mqueue, #sessmem{mqueue = MQueue}) -> +% MQueue; +% info(mqueue_len, #sessmem{mqueue = MQueue}) -> +% emqx_mqueue:len(MQueue); +% info(mqueue_max, #sessmem{mqueue = MQueue}) -> +% emqx_mqueue:max_len(MQueue); +% info(mqueue_dropped, #sessmem{mqueue = MQueue}) -> +% emqx_mqueue:dropped(MQueue); +info(next_pkt_id, #{}) -> + _PacketId = 'TODO'; +% info(awaiting_rel, #sessmem{awaiting_rel = AwaitingRel}) -> +% AwaitingRel; +% info(awaiting_rel_cnt, #sessmem{awaiting_rel = AwaitingRel}) -> +% maps:size(AwaitingRel); +info(awaiting_rel_max, #{props := Conf}) -> + maps:get(max_awaiting_rel, Conf); +info(await_rel_timeout, #{props := Conf}) -> + maps:get(await_rel_timeout, Conf). + +-spec stats(session()) -> emqx_types:stats(). +stats(Session) -> + % TODO: stub + info([], Session). + +%%-------------------------------------------------------------------- +%% Client -> Broker: SUBSCRIBE / UNSUBSCRIBE +%%-------------------------------------------------------------------- + +-spec subscribe(topic(), emqx_types:subopts(), session()) -> + {ok, session()} | {error, emqx_types:reason_code()}. +subscribe( + TopicFilter, + SubOpts, + Session = #{id := ID, iterators := Iters} +) when is_map_key(TopicFilter, Iters) -> + Iterator = maps:get(TopicFilter, Iters), + NIterator = update_subscription(TopicFilter, Iterator, SubOpts, ID), + {ok, Session#{iterators := Iters#{TopicFilter => NIterator}}}; +subscribe( + TopicFilter, + SubOpts, + Session = #{id := ID, iterators := Iters} +) -> + % TODO: max_subscriptions + Iterator = add_subscription(TopicFilter, SubOpts, ID), + {ok, Session#{iterators := Iters#{TopicFilter => Iterator}}}. + +-spec unsubscribe(topic(), session()) -> + {ok, session(), emqx_types:subopts()} | {error, emqx_types:reason_code()}. +unsubscribe( + TopicFilter, + Session = #{id := ID, iterators := Iters} +) when is_map_key(TopicFilter, Iters) -> + Iterator = maps:get(TopicFilter, Iters), + SubOpts = maps:get(props, Iterator), + ok = del_subscription(TopicFilter, Iterator, ID), + {ok, Session#{iterators := maps:remove(TopicFilter, Iters)}, SubOpts}; +unsubscribe( + _TopicFilter, + _Session = #{} +) -> + {error, ?RC_NO_SUBSCRIPTION_EXISTED}. + +-spec get_subscription(topic(), session()) -> + emqx_types:subopts() | undefined. +get_subscription(TopicFilter, #{iterators := Iters}) -> + case maps:get(TopicFilter, Iters, undefined) of + Iterator = #{} -> + maps:get(props, Iterator); + undefined -> + undefined + end. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBLISH +%%-------------------------------------------------------------------- + +-spec publish(emqx_types:packet_id(), emqx_types:message(), session()) -> + {ok, emqx_types:publish_result(), replies(), session()} + | {error, emqx_types:reason_code()}. +publish(_PacketId, Msg, Session) -> + % TODO: stub + {ok, emqx_broker:publish(Msg), [], Session}. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBACK +%%-------------------------------------------------------------------- + +-spec puback(clientinfo(), emqx_types:packet_id(), session()) -> + {ok, emqx_types:message(), replies(), session()} + | {error, emqx_types:reason_code()}. +puback(_ClientInfo, _PacketId, _Session = #{}) -> + % TODO: stub + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBREC +%%-------------------------------------------------------------------- + +-spec pubrec(emqx_types:packet_id(), session()) -> + {ok, emqx_types:message(), session()} + | {error, emqx_types:reason_code()}. +pubrec(_PacketId, _Session = #{}) -> + % TODO: stub + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBREL +%%-------------------------------------------------------------------- + +-spec pubrel(emqx_types:packet_id(), session()) -> + {ok, session()} | {error, emqx_types:reason_code()}. +pubrel(_PacketId, Session = #{}) -> + % TODO: stub + {ok, Session}. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBCOMP +%%-------------------------------------------------------------------- + +-spec pubcomp(clientinfo(), emqx_types:packet_id(), session()) -> + {ok, emqx_types:message(), replies(), session()} + | {error, emqx_types:reason_code()}. +pubcomp(_ClientInfo, _PacketId, _Session = #{}) -> + % TODO: stub + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}. + +%%-------------------------------------------------------------------- + +-spec deliver(clientinfo(), [emqx_types:deliver()], session()) -> + no_return(). +deliver(_ClientInfo, _Delivers, _Session = #{}) -> + % TODO: ensure it's unreachable somehow + error(unexpected). + +-spec replay(clientinfo(), [], session()) -> + {ok, replies(), session()}. +replay(_ClientInfo, [], Session = #{}) -> + {ok, [], Session}. + +%%-------------------------------------------------------------------- + +-spec disconnect(session()) -> {shutdown, session()}. +disconnect(Session = #{}) -> + {shutdown, Session}. + +-spec terminate(Reason :: term(), session()) -> ok. +terminate(_Reason, _Session = #{}) -> + % TODO: close iterators + ok. + +%%-------------------------------------------------------------------- + +-spec add_subscription(topic(), emqx_types:subopts(), id()) -> + emqx_ds:iterator(). +add_subscription(TopicFilterBin, SubOpts, DSSessionID) -> + % N.B.: we chose to update the router before adding the subscription to the + % session/iterator table. The reasoning for this is as follows: + % + % Messages matching this topic filter should start to be persisted as soon as + % possible to avoid missing messages. If this is the first such persistent + % session subscription, it's important to do so early on. + % + % This could, in turn, lead to some inconsistency: if such a route gets + % created but the session/iterator data fails to be updated accordingly, we + % have a dangling route. To remove such dangling routes, we may have a + % periodic GC process that removes routes that do not have a matching + % persistent subscription. Also, route operations use dirty mnesia + % operations, which inherently have room for inconsistencies. + % + % In practice, we use the iterator reference table as a source of truth, + % since it is guarded by a transaction context: we consider a subscription + % operation to be successful if it ended up changing this table. Both router + % and iterator information can be reconstructed from this table, if needed. + ok = emqx_persistent_session_ds_router:do_add_route(TopicFilterBin, DSSessionID), + TopicFilter = emqx_topic:words(TopicFilterBin), + {ok, Iterator, IsNew} = emqx_ds:session_add_iterator( + DSSessionID, TopicFilter, SubOpts + ), + Ctx = #{iterator => Iterator, is_new => IsNew}, + ?tp(persistent_session_ds_iterator_added, Ctx), + ?tp_span( + persistent_session_ds_open_iterators, + Ctx, + ok = open_iterator_on_all_shards(TopicFilter, Iterator) + ), + Iterator. + +-spec update_subscription(topic(), iterator(), emqx_types:subopts(), id()) -> + iterator(). +update_subscription(TopicFilterBin, Iterator, SubOpts, DSSessionID) -> + TopicFilter = emqx_topic:words(TopicFilterBin), + {ok, NIterator, false} = emqx_ds:session_add_iterator( + DSSessionID, TopicFilter, SubOpts + ), + ok = ?tp(persistent_session_ds_iterator_updated, #{iterator => Iterator}), + NIterator. + +-spec open_iterator_on_all_shards(emqx_types:words(), emqx_ds:iterator()) -> ok. +open_iterator_on_all_shards(TopicFilter, Iterator) -> + ?tp(persistent_session_ds_will_open_iterators, #{iterator => Iterator}), %% Note: currently, shards map 1:1 to nodes, but this will change in the future. Nodes = emqx:running_nodes(), Results = emqx_persistent_session_ds_proto_v1:open_iterator( - Nodes, TopicFilter, StartMS, IteratorID + Nodes, + TopicFilter, + maps:get(start_time, Iterator), + maps:get(id, Iterator) ), - %% TODO: handle errors - true = lists:all(fun(Res) -> Res =:= {ok, ok} end, Results), + %% TODO + %% 1. Handle errors. + %% 2. Iterator handles are rocksdb resources, it's doubtful they survive RPC. + %% Even if they do, we throw them away here anyway. All in all, we probably should + %% hold each of them in a process on the respective node. + true = lists:all(fun(Res) -> element(1, Res) =:= ok end, Results), ok. %% RPC target. --spec do_open_iterator(emqx_topic:words(), emqx_ds:time(), emqx_ds:iterator_id()) -> ok. +-spec do_open_iterator(emqx_types:words(), emqx_ds:time(), emqx_ds:iterator_id()) -> + {ok, emqx_ds_storage_layer:iterator()} | {error, _Reason}. do_open_iterator(TopicFilter, StartMS, IteratorID) -> Replay = {TopicFilter, StartMS}, - {ok, _It} = emqx_ds_storage_layer:ensure_iterator(?DS_SHARD, IteratorID, Replay), - ok. + emqx_ds_storage_layer:ensure_iterator(?DS_SHARD, IteratorID, Replay). --spec del_subscription(emqx_types:topic(), emqx_ds:session_id()) -> - ok | {skipped, disabled}. -del_subscription(TopicFilterBin, DSSessionID) -> - ?WHEN_ENABLED( - begin - %% N.B.: see comments in `?MODULE:add_subscription' for a discussion about the - %% order of operations here. - TopicFilter = emqx_topic:words(TopicFilterBin), - case emqx_ds:session_get_iterator_id(DSSessionID, TopicFilter) of - {error, not_found} -> - %% already gone - ok; - {ok, IteratorID} -> - ?tp_span( - persistent_session_ds_close_iterators, - #{iterator_id => IteratorID}, - ok = ensure_iterator_closed_on_all_shards(IteratorID) - ) - end, - ?tp_span( - persistent_session_ds_iterator_delete, - #{}, - emqx_ds:session_del_iterator(DSSessionID, TopicFilter) - ), - ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionID), - ok - end - ). +-spec del_subscription(topic(), iterator(), id()) -> + ok. +del_subscription(TopicFilterBin, #{id := IteratorID}, DSSessionID) -> + % N.B.: see comments in `?MODULE:add_subscription' for a discussion about the + % order of operations here. + TopicFilter = emqx_topic:words(TopicFilterBin), + Ctx = #{iterator_id => IteratorID}, + ?tp_span( + persistent_session_ds_close_iterators, + Ctx, + ok = ensure_iterator_closed_on_all_shards(IteratorID) + ), + ?tp_span( + persistent_session_ds_iterator_delete, + Ctx, + emqx_ds:session_del_iterator(DSSessionID, TopicFilter) + ), + ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilterBin, DSSessionID). -spec ensure_iterator_closed_on_all_shards(emqx_ds:iterator_id()) -> ok. ensure_iterator_closed_on_all_shards(IteratorID) -> @@ -216,7 +434,7 @@ do_ensure_iterator_closed(IteratorID) -> ok = emqx_ds_storage_layer:discard_iterator(?DS_SHARD, IteratorID), ok. --spec ensure_all_iterators_closed(emqx_ds:session_id()) -> ok. +-spec ensure_all_iterators_closed(id()) -> ok. ensure_all_iterators_closed(DSSessionID) -> %% Note: currently, shards map 1:1 to nodes, but this will change in the future. Nodes = emqx:running_nodes(), @@ -226,20 +444,7 @@ ensure_all_iterators_closed(DSSessionID) -> ok. %% RPC target. --spec do_ensure_all_iterators_closed(emqx_ds:session_id()) -> ok. +-spec do_ensure_all_iterators_closed(id()) -> ok. do_ensure_all_iterators_closed(DSSessionID) -> ok = emqx_ds_storage_layer:discard_iterator_prefix(?DS_SHARD, DSSessionID), ok. - -%% - -serialize_message(Msg) -> - term_to_binary(emqx_message:to_map(Msg)). - -deserialize_message(Bin) -> - emqx_message:from_map(binary_to_term(Bin)). - -%% - -is_store_enabled() -> - emqx_config:get([persistent_session_store, ds]). diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index ce71ade91..092c4483a 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -43,11 +43,11 @@ %% MQTT Session -module(emqx_session). +-include("logger.hrl"). +-include("types.hrl"). -include("emqx.hrl"). -include("emqx_session.hrl"). -include("emqx_mqtt.hrl"). --include("logger.hrl"). --include("types.hrl"). -ifdef(TEST). -compile(export_all). @@ -55,18 +55,16 @@ -endif. -export([ - lookup/1, + create/2, + open/2, destroy/1, - unpersist/1 + destroy/2 ]). --export([init/1, init_and_open/1]). - -export([ info/1, info/2, - stats/1, - obtain_next_pkt_id/1 + stats/1 ]). -export([ @@ -79,312 +77,450 @@ puback/3, pubrec/3, pubrel/3, - pubcomp/3 + pubcomp/3, + replay/3 ]). -export([ deliver/3, - enqueue/3, - dequeue/2, - filter_queue/2, - retry/2, + handle_timeout/3, + disconnect/2, terminate/3 ]). --export([ - takeover/1, - resume/2, - replay/2 -]). +% Foreign session implementations +-export([enrich_delivers/3]). --export([expire/3]). +% Utilities +-export([should_keep/1]). -%% Export for CT --export([set_field/3]). - --type session_id() :: emqx_guid:guid(). +% Tests only +-export([get_session_conf/2]). -export_type([ - session/0, - session_id/0 + t/0, + conf/0, + conninfo/0, + reply/0, + replies/0, + common_timer_name/0 ]). --type inflight_data_phase() :: wait_ack | wait_comp. +-type session_id() :: _TODO. --record(inflight_data, { - phase :: inflight_data_phase(), - message :: emqx_types:message(), - timestamp :: non_neg_integer() -}). +-type clientinfo() :: emqx_types:clientinfo(). +-type conninfo() :: + emqx_types:conninfo() + | #{ + %% Subset of `emqx_types:conninfo()` properties + receive_maximum => non_neg_integer(), + expiry_interval => non_neg_integer() + }. --type session() :: #session{}. +-type common_timer_name() :: retry_delivery | expire_awaiting_rel. +-type message() :: emqx_types:message(). -type publish() :: {maybe(emqx_types:packet_id()), emqx_types:message()}. - -type pubrel() :: {pubrel, emqx_types:packet_id()}. +-type reply() :: publish() | pubrel(). +-type replies() :: [reply()] | reply(). --type replies() :: list(publish() | pubrel()). +-type conf() :: #{ + %% Max subscriptions allowed + max_subscriptions := non_neg_integer() | infinity, + %% Max inflight messages allowed + max_inflight := non_neg_integer(), + %% Maximum number of awaiting QoS2 messages allowed + max_awaiting_rel := non_neg_integer() | infinity, + %% Upgrade QoS? + upgrade_qos := boolean(), + %% Retry interval for redelivering QoS1/2 messages (Unit: millisecond) + retry_interval := timeout(), + %% Awaiting PUBREL Timeout (Unit: millisecond) + await_rel_timeout := timeout() +}. + +-type t() :: + emqx_session_mem:session() + | emqx_persistent_session_ds:session(). -define(INFO_KEYS, [ id, + created_at, is_persistent, subscriptions, upgrade_qos, retry_interval, - await_rel_timeout, - created_at + await_rel_timeout ]). --define(STATS_KEYS, [ - subscriptions_cnt, - subscriptions_max, - inflight_cnt, - inflight_max, - mqueue_len, - mqueue_max, - mqueue_dropped, - next_pkt_id, - awaiting_rel_cnt, - awaiting_rel_max -]). - --define(DEFAULT_BATCH_N, 1000). - --type options() :: #{ - max_subscriptions => non_neg_integer(), - upgrade_qos => boolean(), - retry_interval => timeout(), - max_awaiting_rel => non_neg_integer() | infinity, - await_rel_timeout => timeout(), - max_inflight => integer(), - mqueue => emqx_mqueue:options(), - is_persistent => boolean(), - clientid => emqx_types:clientid() -}. +-define(IMPL(S), (get_impl_mod(S))). %%-------------------------------------------------------------------- -%% Init a Session -%%-------------------------------------------------------------------- +%% Behaviour +%% ------------------------------------------------------------------- --spec init_and_open(options()) -> session(). -init_and_open(Options) -> - #{clientid := ClientID} = Options, - Session0 = emqx_session:init(Options), - _ = emqx_persistent_session_ds:open_session(ClientID), - Session0. - --spec init(options()) -> session(). -init(Opts) -> - MaxInflight = maps:get(max_inflight, Opts), - QueueOpts = maps:merge( - #{ - max_len => 1000, - store_qos0 => true - }, - maps:get(mqueue, Opts, #{}) - ), - #session{ - id = emqx_guid:gen(), - clientid = maps:get(clientid, Opts, <<>>), - is_persistent = maps:get(is_persistent, Opts), - max_subscriptions = maps:get(max_subscriptions, Opts), - subscriptions = #{}, - upgrade_qos = maps:get(upgrade_qos, Opts), - inflight = emqx_inflight:new(MaxInflight), - mqueue = emqx_mqueue:init(QueueOpts), - next_pkt_id = 1, - retry_interval = maps:get(retry_interval, Opts), - awaiting_rel = #{}, - max_awaiting_rel = maps:get(max_awaiting_rel, Opts), - await_rel_timeout = maps:get(await_rel_timeout, Opts), - created_at = erlang:system_time(millisecond) - }. - --spec lookup(emqx_types:clientid()) -> none. -lookup(_ClientId) -> - % NOTE - % This is a stub. This session impl has no backing store, thus always `none`. - none. - --spec destroy(emqx_types:clientid()) -> ok. -destroy(_ClientId) -> - % NOTE - % This is a stub. This session impl has no backing store, thus always `ok`. - ok. - --spec unpersist(session()) -> session(). -unpersist(Session) -> - ok = destroy(Session#session.clientid), - Session#session{is_persistent = false}. +-callback create(clientinfo(), conninfo(), conf()) -> + t(). +-callback open(clientinfo(), conninfo()) -> + {_IsPresent :: true, t(), _ReplayContext} | false. %%-------------------------------------------------------------------- -%% Info, Stats +%% Create a Session %%-------------------------------------------------------------------- -%% @doc Get infos of the session. --spec info(session()) -> emqx_types:infos(). -info(Session) -> - maps:from_list(info(?INFO_KEYS, Session)). +-spec create(clientinfo(), conninfo()) -> t(). +create(ClientInfo, ConnInfo) -> + Conf = get_session_conf(ClientInfo, ConnInfo), + create(ClientInfo, ConnInfo, Conf). -info(Keys, Session) when is_list(Keys) -> - [{Key, info(Key, Session)} || Key <- Keys]; -info(id, #session{id = Id}) -> - Id; -info(clientid, #session{clientid = ClientId}) -> - ClientId; -info(is_persistent, #session{is_persistent = Bool}) -> - Bool; -info(subscriptions, #session{subscriptions = Subs}) -> - Subs; -info(subscriptions_cnt, #session{subscriptions = Subs}) -> - maps:size(Subs); -info(subscriptions_max, #session{max_subscriptions = MaxSubs}) -> - MaxSubs; -info(upgrade_qos, #session{upgrade_qos = UpgradeQoS}) -> - UpgradeQoS; -info(inflight, #session{inflight = Inflight}) -> - Inflight; -info(inflight_cnt, #session{inflight = Inflight}) -> - emqx_inflight:size(Inflight); -info(inflight_max, #session{inflight = Inflight}) -> - emqx_inflight:max_size(Inflight); -info(retry_interval, #session{retry_interval = Interval}) -> - Interval; -info(mqueue, #session{mqueue = MQueue}) -> - MQueue; -info(mqueue_len, #session{mqueue = MQueue}) -> - emqx_mqueue:len(MQueue); -info(mqueue_max, #session{mqueue = MQueue}) -> - emqx_mqueue:max_len(MQueue); -info(mqueue_dropped, #session{mqueue = MQueue}) -> - emqx_mqueue:dropped(MQueue); -info(next_pkt_id, #session{next_pkt_id = PacketId}) -> - PacketId; -info(awaiting_rel, #session{awaiting_rel = AwaitingRel}) -> - AwaitingRel; -info(awaiting_rel_cnt, #session{awaiting_rel = AwaitingRel}) -> - maps:size(AwaitingRel); -info(awaiting_rel_max, #session{max_awaiting_rel = Max}) -> - Max; -info(await_rel_timeout, #session{await_rel_timeout = Timeout}) -> - Timeout; -info(created_at, #session{created_at = CreatedAt}) -> - CreatedAt. +create(ClientInfo, ConnInfo, Conf) -> + % FIXME error conditions + create(choose_impl_mod(ConnInfo), ClientInfo, ConnInfo, Conf). -%% @doc Get stats of the session. --spec stats(session()) -> emqx_types:stats(). -stats(Session) -> info(?STATS_KEYS, Session). - -%%-------------------------------------------------------------------- -%% Client -> Broker: SUBSCRIBE -%%-------------------------------------------------------------------- - --spec subscribe( - emqx_types:clientinfo(), - emqx_types:topic(), - emqx_types:subopts(), - session() -) -> - {ok, session()} | {error, emqx_types:reason_code()}. -subscribe( - ClientInfo = #{clientid := ClientId}, - TopicFilter, - SubOpts, - Session = #session{subscriptions = Subs} -) -> - IsNew = not maps:is_key(TopicFilter, Subs), - case IsNew andalso is_subscriptions_full(Session) of - false -> - ok = emqx_broker:subscribe(TopicFilter, ClientId, SubOpts), - Session1 = Session#session{subscriptions = maps:put(TopicFilter, SubOpts, Subs)}, - Session2 = add_persistent_subscription(TopicFilter, ClientId, Session1), - ok = emqx_hooks:run( - 'session.subscribed', - [ClientInfo, TopicFilter, SubOpts#{is_new => IsNew}] - ), - {ok, Session2}; - true -> - {error, ?RC_QUOTA_EXCEEDED} - end. - -is_subscriptions_full(#session{max_subscriptions = infinity}) -> - false; -is_subscriptions_full(#session{ - subscriptions = Subs, - max_subscriptions = MaxLimit -}) -> - maps:size(Subs) >= MaxLimit. - --spec add_persistent_subscription(emqx_types:topic(), emqx_types:clientid(), session()) -> - session(). -add_persistent_subscription(_TopicFilterBin, _ClientId, Session = #session{is_persistent = false}) -> - Session; -add_persistent_subscription(TopicFilterBin, ClientId, Session) -> - _ = emqx_persistent_session_ds:add_subscription(TopicFilterBin, ClientId), +create(Mod, ClientInfo, ConnInfo, Conf) -> + % FIXME error conditions + Session = Mod:create(ClientInfo, ConnInfo, Conf), + ok = emqx_metrics:inc('session.created'), + ok = emqx_hooks:run('session.created', [ClientInfo, info(Session)]), Session. -%%-------------------------------------------------------------------- -%% Client -> Broker: UNSUBSCRIBE -%%-------------------------------------------------------------------- +-spec open(clientinfo(), conninfo()) -> + {_IsPresent :: true, t(), _ReplayContext} | {_IsPresent :: false, t()}. +open(ClientInfo, ConnInfo) -> + Conf = get_session_conf(ClientInfo, ConnInfo), + Mods = [Default | _] = choose_impl_candidates(ConnInfo), + %% NOTE + %% Try to look the existing session up in session stores corresponding to the given + %% `Mods` in order, starting from the last one. + case try_open(Mods, ClientInfo, ConnInfo) of + {_IsPresent = true, _, _} = Present -> + Present; + false -> + %% NOTE + %% Nothing was found, create a new session with the `Default` implementation. + {false, create(Default, ClientInfo, ConnInfo, Conf)} + end. --spec unsubscribe(emqx_types:clientinfo(), emqx_types:topic(), emqx_types:subopts(), session()) -> - {ok, session()} | {error, emqx_types:reason_code()}. +try_open([Mod | Rest], ClientInfo, ConnInfo) -> + case try_open(Rest, ClientInfo, ConnInfo) of + {_IsPresent = true, _, _} = Present -> + Present; + false -> + Mod:open(ClientInfo, ConnInfo) + end; +try_open([], _ClientInfo, _ConnInfo) -> + false. + +-spec get_session_conf(clientinfo(), conninfo()) -> conf(). +get_session_conf( + #{zone := Zone}, + #{receive_maximum := MaxInflight} +) -> + #{ + max_subscriptions => get_mqtt_conf(Zone, max_subscriptions), + max_inflight => MaxInflight, + max_awaiting_rel => get_mqtt_conf(Zone, max_awaiting_rel), + upgrade_qos => get_mqtt_conf(Zone, upgrade_qos), + retry_interval => get_mqtt_conf(Zone, retry_interval), + await_rel_timeout => get_mqtt_conf(Zone, await_rel_timeout) + }. + +get_mqtt_conf(Zone, Key) -> + emqx_config:get_zone_conf(Zone, [mqtt, Key]). + +%%-------------------------------------------------------------------- +%% Existing sessions +%% ------------------------------------------------------------------- + +-spec destroy(clientinfo(), conninfo()) -> ok. +destroy(ClientInfo, ConnInfo) -> + (choose_impl_mod(ConnInfo)):destroy(ClientInfo). + +-spec destroy(t()) -> ok. +destroy(Session) -> + ?IMPL(Session):destroy(Session). + +%%-------------------------------------------------------------------- +%% Subscriptions +%% ------------------------------------------------------------------- + +-spec subscribe( + clientinfo(), + emqx_types:topic(), + emqx_types:subopts(), + t() +) -> + {ok, t()} | {error, emqx_types:reason_code()}. +subscribe(ClientInfo, TopicFilter, SubOpts, Session) -> + SubOpts0 = ?IMPL(Session):get_subscription(TopicFilter, Session), + case ?IMPL(Session):subscribe(TopicFilter, SubOpts, Session) of + {ok, Session1} -> + ok = emqx_hooks:run( + 'session.subscribed', + [ClientInfo, TopicFilter, SubOpts#{is_new => (SubOpts0 == undefined)}] + ), + {ok, Session1}; + {error, RC} -> + {error, RC} + end. + +-spec unsubscribe( + clientinfo(), + emqx_types:topic(), + emqx_types:subopts(), + t() +) -> + {ok, t()} | {error, emqx_types:reason_code()}. unsubscribe( - ClientInfo = #{clientid := ClientId}, + ClientInfo, TopicFilter, UnSubOpts, - Session0 = #session{subscriptions = Subs} + Session ) -> - case maps:find(TopicFilter, Subs) of - {ok, SubOpts} -> - ok = emqx_broker:unsubscribe(TopicFilter), - Session1 = remove_persistent_subscription(Session0, TopicFilter, ClientId), + case ?IMPL(Session):unsubscribe(TopicFilter, Session) of + {ok, Session1, SubOpts} -> ok = emqx_hooks:run( 'session.unsubscribed', [ClientInfo, TopicFilter, maps:merge(SubOpts, UnSubOpts)] ), - {ok, Session1#session{subscriptions = maps:remove(TopicFilter, Subs)}}; - error -> - {error, ?RC_NO_SUBSCRIPTION_EXISTED} + {ok, Session1}; + {error, RC} -> + {error, RC} end. --spec remove_persistent_subscription(session(), emqx_types:topic(), emqx_types:clientid()) -> - session(). -remove_persistent_subscription(Session, TopicFilterBin, ClientId) -> - _ = emqx_persistent_session_ds:del_subscription(TopicFilterBin, ClientId), - Session. - %%-------------------------------------------------------------------- %% Client -> Broker: PUBLISH %%-------------------------------------------------------------------- --spec publish(emqx_types:clientinfo(), emqx_types:packet_id(), emqx_types:message(), session()) -> - {ok, emqx_types:publish_result(), session()} +-spec publish(clientinfo(), emqx_types:packet_id(), emqx_types:message(), t()) -> + {ok, emqx_types:publish_result(), t()} | {error, emqx_types:reason_code()}. -publish( - _ClientInfo, - PacketId, - Msg = #message{qos = ?QOS_2, timestamp = Ts}, - Session = #session{awaiting_rel = AwaitingRel} -) -> - case is_awaiting_full(Session) of - false -> - case maps:is_key(PacketId, AwaitingRel) of - false -> - Results = emqx_broker:publish(Msg), - AwaitingRel1 = maps:put(PacketId, Ts, AwaitingRel), - {ok, Results, Session#session{awaiting_rel = AwaitingRel1}}; - true -> - drop_qos2_msg(PacketId, Msg, ?RC_PACKET_IDENTIFIER_IN_USE) - end; - true -> - drop_qos2_msg(PacketId, Msg, ?RC_RECEIVE_MAXIMUM_EXCEEDED) - end; -%% Publish QoS0/1 directly -publish(_ClientInfo, _PacketId, Msg, Session) -> - {ok, emqx_broker:publish(Msg), Session}. +publish(_ClientInfo, PacketId, Msg, Session) -> + case ?IMPL(Session):publish(PacketId, Msg, Session) of + {ok, _Result, _Session} = Ok -> + % TODO: only timers are allowed for now + Ok; + {error, RC} = Error when Msg#message.qos =:= ?QOS_2 -> + on_dropped_qos2_msg(PacketId, Msg, RC), + Error; + {error, _} = Error -> + Error + end. -drop_qos2_msg(PacketId, Msg, RC) -> +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBACK +%%-------------------------------------------------------------------- + +-spec puback(clientinfo(), emqx_types:packet_id(), t()) -> + {ok, message(), replies(), t()} + | {error, emqx_types:reason_code()}. +puback(ClientInfo, PacketId, Session) -> + case ?IMPL(Session):puback(ClientInfo, PacketId, Session) of + {ok, Msg, Replies, Session1} = Ok -> + _ = on_delivery_completed(Msg, ClientInfo, Session1), + _ = on_replies_delivery_completed(Replies, ClientInfo, Session1), + Ok; + {error, _} = Error -> + Error + end. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBREC / PUBREL / PUBCOMP +%%-------------------------------------------------------------------- + +-spec pubrec(clientinfo(), emqx_types:packet_id(), t()) -> + {ok, message(), t()} + | {error, emqx_types:reason_code()}. +pubrec(_ClientInfo, PacketId, Session) -> + case ?IMPL(Session):pubrec(PacketId, Session) of + {ok, _Msg, _Session} = Ok -> + Ok; + {error, _} = Error -> + Error + end. + +-spec pubrel(clientinfo(), emqx_types:packet_id(), t()) -> + {ok, t()} + | {error, emqx_types:reason_code()}. +pubrel(_ClientInfo, PacketId, Session) -> + case ?IMPL(Session):pubrel(PacketId, Session) of + {ok, _Session} = Ok -> + Ok; + {error, _} = Error -> + Error + end. + +-spec pubcomp(clientinfo(), emqx_types:packet_id(), t()) -> + {ok, replies(), t()} + | {error, emqx_types:reason_code()}. +pubcomp(ClientInfo, PacketId, Session) -> + case ?IMPL(Session):pubcomp(ClientInfo, PacketId, Session) of + {ok, Msg, Replies, Session1} -> + _ = on_delivery_completed(Msg, ClientInfo, Session1), + _ = on_replies_delivery_completed(Replies, ClientInfo, Session1), + {ok, Replies, Session1}; + {error, _} = Error -> + Error + end. + +%%-------------------------------------------------------------------- + +-spec replay(clientinfo(), _ReplayContext, t()) -> + {ok, replies(), t()}. +replay(ClientInfo, ReplayContext, Session) -> + ?IMPL(Session):replay(ClientInfo, ReplayContext, Session). + +%%-------------------------------------------------------------------- +%% Broker -> Client: Deliver +%%-------------------------------------------------------------------- + +-spec deliver(clientinfo(), [emqx_types:deliver()], t()) -> + {ok, replies(), t()}. +deliver(ClientInfo, Delivers, Session) -> + Messages = enrich_delivers(ClientInfo, Delivers, Session), + ?IMPL(Session):deliver(ClientInfo, Messages, Session). + +%%-------------------------------------------------------------------- + +enrich_delivers(ClientInfo, Delivers, Session) -> + UpgradeQoS = ?IMPL(Session):info(upgrade_qos, Session), + enrich_delivers(ClientInfo, Delivers, UpgradeQoS, Session). + +enrich_delivers(_ClientInfo, [], _UpgradeQoS, _Session) -> + []; +enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) -> + case enrich_deliver(ClientInfo, D, UpgradeQoS, Session) of + [] -> + enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session); + Msg -> + [Msg | enrich_delivers(ClientInfo, Rest, UpgradeQoS, Session)] + end. + +enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) -> + SubOpts = ?IMPL(Session):get_subscription(Topic, Session), + enrich_message(ClientInfo, Msg, SubOpts, UpgradeQoS). + +enrich_message( + ClientInfo = #{clientid := ClientId}, + Msg = #message{from = ClientId}, + #{nl := 1}, + _UpgradeQoS +) -> + _ = emqx_session_events:handle_event(ClientInfo, {dropped, Msg, no_local}), + []; +enrich_message(_ClientInfo, MsgIn, SubOpts = #{}, UpgradeQoS) -> + maps:fold( + fun(SubOpt, V, Msg) -> enrich_subopts(SubOpt, V, Msg, UpgradeQoS) end, + MsgIn, + SubOpts + ); +enrich_message(_ClientInfo, Msg, undefined, _UpgradeQoS) -> + Msg. + +enrich_subopts(nl, 1, Msg, _) -> + emqx_message:set_flag(nl, Msg); +enrich_subopts(nl, 0, Msg, _) -> + Msg; +enrich_subopts(qos, SubQoS, Msg = #message{qos = PubQoS}, _UpgradeQoS = true) -> + Msg#message{qos = max(SubQoS, PubQoS)}; +enrich_subopts(qos, SubQoS, Msg = #message{qos = PubQoS}, _UpgradeQoS = false) -> + Msg#message{qos = min(SubQoS, PubQoS)}; +enrich_subopts(rap, 1, Msg, _) -> + Msg; +enrich_subopts(rap, 0, Msg = #message{headers = #{retained := true}}, _) -> + Msg; +enrich_subopts(rap, 0, Msg, _) -> + emqx_message:set_flag(retain, false, Msg); +enrich_subopts(subid, SubId, Msg, _) -> + Props = emqx_message:get_header(properties, Msg, #{}), + emqx_message:set_header(properties, Props#{'Subscription-Identifier' => SubId}, Msg); +enrich_subopts(_Opt, _V, Msg, _) -> + Msg. + +%%-------------------------------------------------------------------- +%% Timeouts +%%-------------------------------------------------------------------- + +-spec handle_timeout(clientinfo(), common_timer_name(), t()) -> + {ok, replies(), t()} + | {ok, replies(), timeout(), t()}. +handle_timeout(ClientInfo, Timer, Session) -> + ?IMPL(Session):handle_timeout(ClientInfo, Timer, Session). + +%%-------------------------------------------------------------------- + +-spec disconnect(clientinfo(), t()) -> + {idle | shutdown, t()}. +disconnect(_ClientInfo, Session) -> + ?IMPL(Session):disconnect(Session). + +-spec terminate(clientinfo(), Reason :: term(), t()) -> + ok. +terminate(ClientInfo, Reason, Session) -> + _ = run_terminate_hooks(ClientInfo, Reason, Session), + _ = ?IMPL(Session):terminate(Reason, Session), + ok. + +run_terminate_hooks(ClientInfo, discarded, Session) -> + run_hook('session.discarded', [ClientInfo, info(Session)]); +run_terminate_hooks(ClientInfo, takenover, Session) -> + run_hook('session.takenover', [ClientInfo, info(Session)]); +run_terminate_hooks(ClientInfo, Reason, Session) -> + run_hook('session.terminated', [ClientInfo, Reason, info(Session)]). + +%%-------------------------------------------------------------------- +%% Session Info +%% ------------------------------------------------------------------- + +-spec info(t()) -> emqx_types:infos(). +info(Session) -> + maps:from_list(info(?INFO_KEYS, Session)). + +-spec info + ([atom()], t()) -> [{atom(), _Value}]; + (atom(), t()) -> _Value. +info(Keys, Session) when is_list(Keys) -> + [{Key, info(Key, Session)} || Key <- Keys]; +info(impl, Session) -> + get_impl_mod(Session); +info(Key, Session) -> + ?IMPL(Session):info(Key, Session). + +-spec stats(t()) -> emqx_types:stats(). +stats(Session) -> + ?IMPL(Session):stats(Session). + +%%-------------------------------------------------------------------- +%% Common message events +%%-------------------------------------------------------------------- + +on_delivery_completed(Msg, #{clientid := ClientId}, Session) -> + emqx:run_hook( + 'delivery.completed', + [ + Msg, + #{ + session_birth_time => ?IMPL(Session):info(created_at, Session), + clientid => ClientId + } + ] + ). + +on_replies_delivery_completed(Replies, ClientInfo, Session) -> + lists:foreach( + fun({_PacketId, Msg}) -> + case Msg of + #message{qos = ?QOS_0} -> + on_delivery_completed(Msg, ClientInfo, Session); + _ -> + ok + end + end, + Replies + ). + +on_dropped_qos2_msg(PacketId, Msg, RC) -> ?SLOG( warning, #{ @@ -396,550 +532,46 @@ drop_qos2_msg(PacketId, Msg, RC) -> ), ok = emqx_metrics:inc('messages.dropped'), ok = emqx_hooks:run('message.dropped', [Msg, #{node => node()}, emqx_reason_codes:name(RC)]), - {error, RC}. - -is_awaiting_full(#session{max_awaiting_rel = infinity}) -> - false; -is_awaiting_full(#session{ - awaiting_rel = AwaitingRel, - max_awaiting_rel = MaxLimit -}) -> - maps:size(AwaitingRel) >= MaxLimit. - -%%-------------------------------------------------------------------- -%% Client -> Broker: PUBACK -%%-------------------------------------------------------------------- - --spec puback(emqx_types:clientinfo(), emqx_types:packet_id(), session()) -> - {ok, emqx_types:message(), session()} - | {ok, emqx_types:message(), replies(), session()} - | {error, emqx_types:reason_code()}. -puback(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> - case emqx_inflight:lookup(PacketId, Inflight) of - {value, #inflight_data{phase = wait_ack, message = Msg}} -> - on_delivery_completed(Msg, Session), - Inflight1 = emqx_inflight:delete(PacketId, Inflight), - return_with(Msg, dequeue(ClientInfo, Session#session{inflight = Inflight1})); - {value, _} -> - {error, ?RC_PACKET_IDENTIFIER_IN_USE}; - none -> - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} - end. - -return_with(Msg, {ok, Session}) -> - {ok, Msg, Session}; -return_with(Msg, {ok, Publishes, Session}) -> - {ok, Msg, Publishes, Session}. - -%%-------------------------------------------------------------------- -%% Client -> Broker: PUBREC -%%-------------------------------------------------------------------- - --spec pubrec(emqx_types:clientinfo(), emqx_types:packet_id(), session()) -> - {ok, emqx_types:message(), session()} - | {error, emqx_types:reason_code()}. -pubrec(_ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> - case emqx_inflight:lookup(PacketId, Inflight) of - {value, #inflight_data{phase = wait_ack, message = Msg} = Data} -> - Update = Data#inflight_data{phase = wait_comp}, - Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), - {ok, Msg, Session#session{inflight = Inflight1}}; - {value, _} -> - {error, ?RC_PACKET_IDENTIFIER_IN_USE}; - none -> - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} - end. - -%%-------------------------------------------------------------------- -%% Client -> Broker: PUBREL -%%-------------------------------------------------------------------- - --spec pubrel(emqx_types:clientinfo(), emqx_types:packet_id(), session()) -> - {ok, session()} | {error, emqx_types:reason_code()}. -pubrel(_ClientInfo, PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> - case maps:take(PacketId, AwaitingRel) of - {_Ts, AwaitingRel1} -> - {ok, Session#session{awaiting_rel = AwaitingRel1}}; - error -> - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} - end. - -%%-------------------------------------------------------------------- -%% Client -> Broker: PUBCOMP -%%-------------------------------------------------------------------- - --spec pubcomp(emqx_types:clientinfo(), emqx_types:packet_id(), session()) -> - {ok, session()} - | {ok, replies(), session()} - | {error, emqx_types:reason_code()}. -pubcomp(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> - case emqx_inflight:lookup(PacketId, Inflight) of - {value, #inflight_data{phase = wait_comp, message = Msg}} -> - on_delivery_completed(Msg, Session), - Inflight1 = emqx_inflight:delete(PacketId, Inflight), - dequeue(ClientInfo, Session#session{inflight = Inflight1}); - {value, _Other} -> - {error, ?RC_PACKET_IDENTIFIER_IN_USE}; - none -> - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} - end. - -%%-------------------------------------------------------------------- -%% Dequeue Msgs -%%-------------------------------------------------------------------- - -dequeue(ClientInfo, Session = #session{inflight = Inflight, mqueue = Q}) -> - case emqx_mqueue:is_empty(Q) of - true -> - {ok, Session}; - false -> - {Msgs, Q1} = dequeue(ClientInfo, batch_n(Inflight), [], Q), - do_deliver(ClientInfo, Msgs, [], Session#session{mqueue = Q1}) - end. - -dequeue(_ClientInfo, 0, Msgs, Q) -> - {lists:reverse(Msgs), Q}; -dequeue(ClientInfo, Cnt, Msgs, Q) -> - case emqx_mqueue:out(Q) of - {empty, _Q} -> - dequeue(ClientInfo, 0, Msgs, Q); - {{value, Msg}, Q1} -> - case emqx_message:is_expired(Msg) of - true -> - ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, expired]), - ok = inc_delivery_expired_cnt(), - dequeue(ClientInfo, Cnt, Msgs, Q1); - false -> - dequeue(ClientInfo, acc_cnt(Msg, Cnt), [Msg | Msgs], Q1) - end - end. - -filter_queue(Pred, #session{mqueue = Q} = Session) -> - Session#session{mqueue = emqx_mqueue:filter(Pred, Q)}. - -acc_cnt(#message{qos = ?QOS_0}, Cnt) -> Cnt; -acc_cnt(_Msg, Cnt) -> Cnt - 1. - -%%-------------------------------------------------------------------- -%% Broker -> Client: Deliver -%%-------------------------------------------------------------------- - --spec deliver(emqx_types:clientinfo(), list(emqx_types:deliver()), session()) -> - {ok, session()} | {ok, replies(), session()}. -%% Optimize -deliver(ClientInfo, [Deliver], Session) -> - Msg = enrich_deliver(Deliver, Session), - deliver_msg(ClientInfo, Msg, Session); -deliver(ClientInfo, Delivers, Session) -> - Msgs = [enrich_deliver(D, Session) || D <- Delivers], - do_deliver(ClientInfo, Msgs, [], Session). - -do_deliver(_ClientInfo, [], Publishes, Session) -> - {ok, lists:reverse(Publishes), Session}; -do_deliver(ClientInfo, [Msg | More], Acc, Session) -> - case deliver_msg(ClientInfo, Msg, Session) of - {ok, Session1} -> - do_deliver(ClientInfo, More, Acc, Session1); - {ok, [Publish], Session1} -> - do_deliver(ClientInfo, More, [Publish | Acc], Session1) - end. - -deliver_msg(_ClientInfo, Msg = #message{qos = ?QOS_0}, Session) -> - % - on_delivery_completed(Msg, Session), - {ok, [{undefined, maybe_ack(Msg)}], Session}; -deliver_msg( - ClientInfo, - Msg = #message{qos = QoS}, - Session = - #session{next_pkt_id = PacketId, inflight = Inflight} -) when - QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 --> - case emqx_inflight:is_full(Inflight) of - true -> - Session1 = - case maybe_nack(Msg) of - true -> Session; - false -> enqueue(ClientInfo, Msg, Session) - end, - {ok, Session1}; - false -> - %% Note that we publish message without shared ack header - %% But add to inflight with ack headers - %% This ack header is required for redispatch-on-terminate feature to work - Publish = {PacketId, maybe_ack(Msg)}, - MarkedMsg = mark_begin_deliver(Msg), - Inflight1 = emqx_inflight:insert(PacketId, with_ts(MarkedMsg), Inflight), - {ok, [Publish], next_pkt_id(Session#session{inflight = Inflight1})} - end; -deliver_msg(ClientInfo, {drop, Msg, Reason}, Session) -> - handle_dropped(ClientInfo, Msg, Reason, Session), - {ok, Session}. - --spec enqueue( - emqx_types:clientinfo(), - list(emqx_types:deliver()) | emqx_types:message(), - session() -) -> session(). -enqueue(ClientInfo, Delivers, Session) when is_list(Delivers) -> - lists:foldl( - fun(Deliver, Session0) -> - Msg = enrich_deliver(Deliver, Session), - enqueue(ClientInfo, Msg, Session0) - end, - Session, - Delivers - ); -enqueue(ClientInfo, #message{} = Msg, Session = #session{mqueue = Q}) -> - {Dropped, NewQ} = emqx_mqueue:in(Msg, Q), - (Dropped =/= undefined) andalso handle_dropped(ClientInfo, Dropped, Session), - Session#session{mqueue = NewQ}; -enqueue(ClientInfo, {drop, Msg, Reason}, Session) -> - handle_dropped(ClientInfo, Msg, Reason, Session), - Session. - -handle_dropped(ClientInfo, Msg = #message{qos = QoS, topic = Topic}, #session{mqueue = Q}) -> - Payload = emqx_message:to_log_map(Msg), - #{store_qos0 := StoreQos0} = QueueInfo = emqx_mqueue:info(Q), - case (QoS == ?QOS_0) andalso (not StoreQos0) of - true -> - ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, qos0_msg]), - ok = emqx_metrics:inc('delivery.dropped'), - ok = emqx_metrics:inc('delivery.dropped.qos0_msg'), - ok = inc_pd('send_msg.dropped'), - ?SLOG( - warning, - #{ - msg => "dropped_qos0_msg", - queue => QueueInfo, - payload => Payload - }, - #{topic => Topic} - ); - false -> - ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, queue_full]), - ok = emqx_metrics:inc('delivery.dropped'), - ok = emqx_metrics:inc('delivery.dropped.queue_full'), - ok = inc_pd('send_msg.dropped'), - ok = inc_pd('send_msg.dropped.queue_full'), - ?SLOG( - warning, - #{ - msg => "dropped_msg_due_to_mqueue_is_full", - queue => QueueInfo, - payload => Payload - }, - #{topic => Topic} - ) - end. - -handle_dropped(ClientInfo, Msg, Reason, _Session) -> - ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, Reason]), - ok = emqx_metrics:inc('delivery.dropped'), - ok = emqx_metrics:inc('delivery.dropped.no_local'). - -enrich_deliver({deliver, Topic, Msg}, Session = #session{subscriptions = Subs}) -> - enrich_deliver(Msg, maps:find(Topic, Subs), Session). - -enrich_deliver(Msg = #message{from = ClientId}, {ok, #{nl := 1}}, #session{clientid = ClientId}) -> - {drop, Msg, no_local}; -enrich_deliver(Msg, SubOpts, Session) -> - enrich_subopts(mk_subopts(SubOpts), Msg, Session). - -maybe_ack(Msg) -> - emqx_shared_sub:maybe_ack(Msg). - -maybe_nack(Msg) -> - emqx_shared_sub:maybe_nack_dropped(Msg). - -mk_subopts(SubOpts) -> - case SubOpts of - {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} -> - [{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}]; - {ok, #{nl := Nl, qos := QoS, rap := Rap}} -> - [{nl, Nl}, {qos, QoS}, {rap, Rap}]; - error -> - [] - end. - -enrich_subopts([], Msg, _Session) -> - Msg; -enrich_subopts([{nl, 1} | Opts], Msg, Session) -> - enrich_subopts(Opts, emqx_message:set_flag(nl, Msg), Session); -enrich_subopts([{nl, 0} | Opts], Msg, Session) -> - enrich_subopts(Opts, Msg, Session); -enrich_subopts( - [{qos, SubQoS} | Opts], - Msg = #message{qos = PubQoS}, - Session = #session{upgrade_qos = true} -) -> - enrich_subopts(Opts, Msg#message{qos = max(SubQoS, PubQoS)}, Session); -enrich_subopts( - [{qos, SubQoS} | Opts], - Msg = #message{qos = PubQoS}, - Session = #session{upgrade_qos = false} -) -> - enrich_subopts(Opts, Msg#message{qos = min(SubQoS, PubQoS)}, Session); -enrich_subopts([{rap, 1} | Opts], Msg, Session) -> - enrich_subopts(Opts, Msg, Session); -enrich_subopts([{rap, 0} | Opts], Msg = #message{headers = #{retained := true}}, Session) -> - enrich_subopts(Opts, Msg, Session); -enrich_subopts([{rap, 0} | Opts], Msg, Session) -> - enrich_subopts(Opts, emqx_message:set_flag(retain, false, Msg), Session); -enrich_subopts([{subid, SubId} | Opts], Msg, Session) -> - Props = emqx_message:get_header(properties, Msg, #{}), - Msg1 = emqx_message:set_header(properties, Props#{'Subscription-Identifier' => SubId}, Msg), - enrich_subopts(Opts, Msg1, Session). - -%%-------------------------------------------------------------------- -%% Retry Delivery -%%-------------------------------------------------------------------- - --spec retry(emqx_types:clientinfo(), session()) -> - {ok, session()} | {ok, replies(), timeout(), session()}. -retry(ClientInfo, Session = #session{inflight = Inflight}) -> - case emqx_inflight:is_empty(Inflight) of - true -> - {ok, Session}; - false -> - Now = erlang:system_time(millisecond), - retry_delivery( - emqx_inflight:to_list(fun sort_fun/2, Inflight), - [], - Now, - Session, - ClientInfo - ) - end. - -retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}, _ClientInfo) -> - {ok, lists:reverse(Acc), Interval, Session}; -retry_delivery( - [{PacketId, #inflight_data{timestamp = Ts} = Data} | More], - Acc, - Now, - Session = #session{retry_interval = Interval, inflight = Inflight}, - ClientInfo -) -> - case (Age = age(Now, Ts)) >= Interval of - true -> - {Acc1, Inflight1} = do_retry_delivery(PacketId, Data, Now, Acc, Inflight, ClientInfo), - retry_delivery(More, Acc1, Now, Session#session{inflight = Inflight1}, ClientInfo); - false -> - {ok, lists:reverse(Acc), Interval - max(0, Age), Session} - end. - -do_retry_delivery( - PacketId, - #inflight_data{phase = wait_ack, message = Msg} = Data, - Now, - Acc, - Inflight, - ClientInfo -) -> - case emqx_message:is_expired(Msg) of - true -> - ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, expired]), - ok = inc_delivery_expired_cnt(), - {Acc, emqx_inflight:delete(PacketId, Inflight)}; - false -> - Msg1 = emqx_message:set_flag(dup, true, Msg), - Update = Data#inflight_data{message = Msg1, timestamp = Now}, - Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), - {[{PacketId, Msg1} | Acc], Inflight1} - end; -do_retry_delivery(PacketId, Data, Now, Acc, Inflight, _) -> - Update = Data#inflight_data{timestamp = Now}, - Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), - {[{pubrel, PacketId} | Acc], Inflight1}. - -%%-------------------------------------------------------------------- -%% Expire Awaiting Rel -%%-------------------------------------------------------------------- - --spec expire(emqx_types:clientinfo(), awaiting_rel, session()) -> - {ok, session()} | {ok, timeout(), session()}. -expire(_ClientInfo, awaiting_rel, Session = #session{awaiting_rel = AwaitingRel}) -> - case maps:size(AwaitingRel) of - 0 -> {ok, Session}; - _ -> expire_awaiting_rel(erlang:system_time(millisecond), Session) - end. - -expire_awaiting_rel( - Now, - Session = #session{ - awaiting_rel = AwaitingRel, - await_rel_timeout = Timeout - } -) -> - NotExpired = fun(_PacketId, Ts) -> age(Now, Ts) < Timeout end, - AwaitingRel1 = maps:filter(NotExpired, AwaitingRel), - ExpiredCnt = maps:size(AwaitingRel) - maps:size(AwaitingRel1), - (ExpiredCnt > 0) andalso inc_await_pubrel_timeout(ExpiredCnt), - NSession = Session#session{awaiting_rel = AwaitingRel1}, - case maps:size(AwaitingRel1) of - 0 -> {ok, NSession}; - _ -> {ok, Timeout, NSession} - end. - -%%-------------------------------------------------------------------- -%% Takeover, Resume and Replay -%%-------------------------------------------------------------------- - --spec takeover(session()) -> ok. -takeover(#session{subscriptions = Subs}) -> - lists:foreach(fun emqx_broker:unsubscribe/1, maps:keys(Subs)). - --spec resume(emqx_types:clientinfo(), session()) -> ok. -resume(ClientInfo = #{clientid := ClientId}, Session = #session{subscriptions = Subs}) -> - lists:foreach( - fun({TopicFilter, SubOpts}) -> - ok = emqx_broker:subscribe(TopicFilter, ClientId, SubOpts) - end, - maps:to_list(Subs) - ), - ok = emqx_metrics:inc('session.resumed'), - emqx_hooks:run('session.resumed', [ClientInfo, info(Session)]). - --spec replay(emqx_types:clientinfo(), session()) -> {ok, replies(), session()}. -replay(ClientInfo, Session = #session{inflight = Inflight}) -> - Pubs = lists:map( - fun - ({PacketId, #inflight_data{phase = wait_comp}}) -> - {pubrel, PacketId}; - ({PacketId, #inflight_data{message = Msg}}) -> - {PacketId, emqx_message:set_flag(dup, true, Msg)} - end, - emqx_inflight:to_list(Inflight) - ), - case dequeue(ClientInfo, Session) of - {ok, NSession} -> {ok, Pubs, NSession}; - {ok, More, NSession} -> {ok, lists:append(Pubs, More), NSession} - end. - --spec terminate(emqx_types:clientinfo(), Reason :: term(), session()) -> ok. -terminate(ClientInfo, Reason, Session) -> - run_terminate_hooks(ClientInfo, Reason, Session), - maybe_redispatch_shared_messages(Reason, Session), ok. -run_terminate_hooks(ClientInfo, discarded, Session) -> - run_hook('session.discarded', [ClientInfo, info(Session)]); -run_terminate_hooks(ClientInfo, takenover, Session) -> - run_hook('session.takenover', [ClientInfo, info(Session)]); -run_terminate_hooks(ClientInfo, Reason, Session) -> - run_hook('session.terminated', [ClientInfo, Reason, info(Session)]). +%%-------------------------------------------------------------------- -maybe_redispatch_shared_messages(takenover, _Session) -> - ok; -maybe_redispatch_shared_messages(kicked, _Session) -> - ok; -maybe_redispatch_shared_messages(_Reason, Session) -> - redispatch_shared_messages(Session). +-spec should_keep(message() | emqx_types:deliver()) -> boolean(). +should_keep(MsgDeliver) -> + not is_banned_msg(MsgDeliver). -redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) -> - AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight), - F = fun - ({_PacketId, #inflight_data{message = #message{qos = ?QOS_1} = Msg}}) -> - %% For QoS 2, here is what the spec says: - %% If the Client's Session terminates before the Client reconnects, - %% the Server MUST NOT send the Application Message to any other - %% subscribed Client [MQTT-4.8.2-5]. - {true, Msg}; - ({_PacketId, #inflight_data{}}) -> - false - end, - InflightList = lists:filtermap(F, AllInflights), - emqx_shared_sub:redispatch(InflightList ++ emqx_mqueue:to_list(Q)). +is_banned_msg(#message{from = ClientId}) -> + [] =/= emqx_banned:look_up({clientid, ClientId}). + +%%-------------------------------------------------------------------- + +-spec get_impl_mod(t()) -> module(). +get_impl_mod(Session) when ?IS_SESSION_IMPL_MEM(Session) -> + emqx_session_mem; +get_impl_mod(Session) when ?IS_SESSION_IMPL_DS(Session) -> + emqx_persistent_session_ds. + +-spec choose_impl_mod(conninfo()) -> module(). +choose_impl_mod(#{expiry_interval := EI}) -> + hd(choose_impl_candidates(EI, emqx_persistent_message:is_store_enabled())). + +-spec choose_impl_candidates(conninfo()) -> [module()]. +choose_impl_candidates(#{expiry_interval := EI}) -> + choose_impl_candidates(EI, emqx_persistent_message:is_store_enabled()). + +choose_impl_candidates(_, _IsPSStoreEnabled = false) -> + [emqx_session_mem]; +choose_impl_candidates(0, _IsPSStoreEnabled = true) -> + %% NOTE + %% If ExpiryInterval is 0, the natural choice is `emqx_session_mem`. Yet we still + %% need to look the existing session up in the `emqx_persistent_session_ds` store + %% first, because previous connection may have set ExpiryInterval to a non-zero + %% value. + [emqx_session_mem, emqx_persistent_session_ds]; +choose_impl_candidates(EI, _IsPSStoreEnabled = true) when EI > 0 -> + [emqx_persistent_session_ds]. -compile({inline, [run_hook/2]}). run_hook(Name, Args) -> ok = emqx_metrics:inc(Name), emqx_hooks:run(Name, Args). - -%%-------------------------------------------------------------------- -%% Inc message/delivery expired counter -%%-------------------------------------------------------------------- -inc_delivery_expired_cnt() -> - inc_delivery_expired_cnt(1). - -inc_delivery_expired_cnt(N) -> - ok = inc_pd('send_msg.dropped', N), - ok = inc_pd('send_msg.dropped.expired', N), - ok = emqx_metrics:inc('delivery.dropped', N), - emqx_metrics:inc('delivery.dropped.expired', N). - -inc_await_pubrel_timeout(N) -> - ok = inc_pd('recv_msg.dropped', N), - ok = inc_pd('recv_msg.dropped.await_pubrel_timeout', N), - ok = emqx_metrics:inc('messages.dropped', N), - emqx_metrics:inc('messages.dropped.await_pubrel_timeout', N). - -inc_pd(Key) -> - inc_pd(Key, 1). -inc_pd(Key, Inc) -> - _ = emqx_pd:inc_counter(Key, Inc), - ok. - -%%-------------------------------------------------------------------- -%% Next Packet Id -%%-------------------------------------------------------------------- - -obtain_next_pkt_id(Session) -> - {Session#session.next_pkt_id, next_pkt_id(Session)}. - -next_pkt_id(Session = #session{next_pkt_id = ?MAX_PACKET_ID}) -> - Session#session{next_pkt_id = 1}; -next_pkt_id(Session = #session{next_pkt_id = Id}) -> - Session#session{next_pkt_id = Id + 1}. - -%%-------------------------------------------------------------------- -%% Message Latency Stats -%%-------------------------------------------------------------------- -on_delivery_completed( - Msg, - #session{created_at = CreateAt, clientid = ClientId} -) -> - emqx:run_hook( - 'delivery.completed', - [ - Msg, - #{session_birth_time => CreateAt, clientid => ClientId} - ] - ). - -mark_begin_deliver(Msg) -> - emqx_message:set_header(deliver_begin_at, erlang:system_time(millisecond), Msg). - -%%-------------------------------------------------------------------- -%% Helper functions -%%-------------------------------------------------------------------- - --compile({inline, [sort_fun/2, batch_n/1, with_ts/1, age/2]}). - -sort_fun({_, A}, {_, B}) -> - A#inflight_data.timestamp =< B#inflight_data.timestamp. - -batch_n(Inflight) -> - case emqx_inflight:max_size(Inflight) of - 0 -> ?DEFAULT_BATCH_N; - Sz -> Sz - emqx_inflight:size(Inflight) - end. - -with_ts(Msg) -> - #inflight_data{ - phase = wait_ack, - message = Msg, - timestamp = erlang:system_time(millisecond) - }. - -age(Now, Ts) -> Now - Ts. - -%%-------------------------------------------------------------------- -%% For CT tests -%%-------------------------------------------------------------------- - -set_field(Name, Value, Session) -> - Pos = emqx_utils:index_of(Name, record_info(fields, session)), - setelement(Pos + 1, Session, Value). diff --git a/apps/emqx/src/emqx_session_events.erl b/apps/emqx/src/emqx_session_events.erl new file mode 100644 index 000000000..b04dd2044 --- /dev/null +++ b/apps/emqx/src/emqx_session_events.erl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_session_events). + +-include("emqx.hrl"). +-include("logger.hrl"). + +-export([handle_event/2]). + +-type message() :: emqx_types:message(). + +-type event_expired() :: {expired, message()}. +-type event_dropped() :: {dropped, message(), _Reason :: atom() | #{reason := atom(), _ => _}}. +-type event_expire_rel() :: {expired_rel, non_neg_integer()}. + +-type event() :: + event_expired() + | event_dropped() + | event_expire_rel(). + +%% + +-spec handle_event(emqx_session:client_info(), event()) -> + ok. +handle_event(ClientInfo, {expired, Msg}) -> + ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, expired]), + ok = inc_delivery_expired_cnt(1); +handle_event(ClientInfo, {dropped, Msg, no_local}) -> + ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, no_local]), + ok = emqx_metrics:inc('delivery.dropped'), + ok = emqx_metrics:inc('delivery.dropped.no_local'); +handle_event(ClientInfo, {dropped, Msg, #{reason := qos0_msg, logctx := Ctx}}) -> + ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, qos0_msg]), + ok = emqx_metrics:inc('delivery.dropped'), + ok = emqx_metrics:inc('delivery.dropped.qos0_msg'), + ok = inc_pd('send_msg.dropped', 1), + ?SLOG( + warning, + Ctx#{ + msg => "dropped_qos0_msg", + payload => Msg#message.payload + }, + #{topic => Msg#message.topic} + ); +handle_event(ClientInfo, {dropped, Msg, #{reason := queue_full, logctx := Ctx}}) -> + ok = emqx_hooks:run('delivery.dropped', [ClientInfo, Msg, queue_full]), + ok = emqx_metrics:inc('delivery.dropped'), + ok = emqx_metrics:inc('delivery.dropped.queue_full'), + ok = inc_pd('send_msg.dropped', 1), + ok = inc_pd('send_msg.dropped.queue_full', 1), + ?SLOG( + warning, + Ctx#{ + msg => "dropped_msg_due_to_mqueue_is_full", + payload => Msg#message.payload + }, + #{topic => Msg#message.topic} + ); +handle_event(_ClientInfo, {expired_rel, 0}) -> + ok; +handle_event(_ClientInfo, {expired_rel, ExpiredCnt}) -> + inc_await_pubrel_timeout(ExpiredCnt). + +inc_delivery_expired_cnt(N) -> + ok = inc_pd('send_msg.dropped', N), + ok = inc_pd('send_msg.dropped.expired', N), + ok = emqx_metrics:inc('delivery.dropped', N), + emqx_metrics:inc('delivery.dropped.expired', N). + +inc_await_pubrel_timeout(N) -> + ok = inc_pd('recv_msg.dropped', N), + ok = inc_pd('recv_msg.dropped.await_pubrel_timeout', N), + ok = emqx_metrics:inc('messages.dropped', N), + emqx_metrics:inc('messages.dropped.await_pubrel_timeout', N). + +inc_pd(Key, Inc) -> + _ = emqx_pd:inc_counter(Key, Inc), + ok. diff --git a/apps/emqx/src/emqx_session_mem.erl b/apps/emqx/src/emqx_session_mem.erl new file mode 100644 index 000000000..e72feffd5 --- /dev/null +++ b/apps/emqx/src/emqx_session_mem.erl @@ -0,0 +1,800 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% @doc +%% A stateful interaction between a Client and a Server. Some Sessions +%% last only as long as the Network Connection, others can span multiple +%% consecutive Network Connections between a Client and a Server. +%% +%% The Session State in the Server consists of: +%% +%% The existence of a Session, even if the rest of the Session State is empty. +%% +%% The Clients subscriptions, including any Subscription Identifiers. +%% +%% QoS 1 and QoS 2 messages which have been sent to the Client, but have not +%% been completely acknowledged. +%% +%% QoS 1 and QoS 2 messages pending transmission to the Client and OPTIONALLY +%% QoS 0 messages pending transmission to the Client. +%% +%% QoS 2 messages which have been received from the Client, but have not been +%% completely acknowledged.The Will Message and the Will Delay Interval +%% +%% If the Session is currently not connected, the time at which the Session +%% will end and Session State will be discarded. +%% @end +%%-------------------------------------------------------------------- + +%% MQTT Session implementation +%% State is stored in-memory in the process heap. +-module(emqx_session_mem). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). +-include("emqx_session_mem.hrl"). +-include("logger.hrl"). +-include("types.hrl"). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +-export([ + create/3, + open/2, + destroy/1 +]). + +-export([ + info/2, + stats/1, + obtain_next_pkt_id/1 +]). + +-export([ + subscribe/3, + unsubscribe/2, + get_subscription/2 +]). + +-export([ + publish/3, + puback/3, + pubrec/2, + pubrel/2, + pubcomp/3 +]). + +-export([ + deliver/3, + replay/3, + handle_timeout/3, + disconnect/1, + terminate/2 +]). + +-export([ + retry/2, + expire/2 +]). + +%% Part of takeover sequence +-export([ + takeover/1, + resume/2, + enqueue/3, + dequeue/2, + replay/2, + dedup/4 +]). + +%% Export for CT +-export([set_field/3]). + +-type session_id() :: emqx_guid:guid(). + +-export_type([ + session/0, + session_id/0 +]). + +-type inflight_data_phase() :: wait_ack | wait_comp. + +-record(inflight_data, { + phase :: inflight_data_phase(), + message :: emqx_types:message(), + timestamp :: non_neg_integer() +}). + +-type session() :: #session{}. +-type replayctx() :: [emqx_types:message()]. + +-type clientinfo() :: emqx_types:clientinfo(). +-type conninfo() :: emqx_session:conninfo(). +-type replies() :: emqx_session:replies(). + +-define(STATS_KEYS, [ + subscriptions_cnt, + subscriptions_max, + inflight_cnt, + inflight_max, + mqueue_len, + mqueue_max, + mqueue_dropped, + next_pkt_id, + awaiting_rel_cnt, + awaiting_rel_max +]). + +-define(DEFAULT_BATCH_N, 1000). + +%%-------------------------------------------------------------------- +%% Init a Session +%%-------------------------------------------------------------------- + +-spec create(clientinfo(), conninfo(), emqx_session:conf()) -> + session(). +create(#{zone := Zone, clientid := ClientId}, #{expiry_interval := EI}, Conf) -> + QueueOpts = get_mqueue_conf(Zone), + #session{ + id = emqx_guid:gen(), + clientid = ClientId, + created_at = erlang:system_time(millisecond), + is_persistent = EI > 0, + subscriptions = #{}, + inflight = emqx_inflight:new(maps:get(max_inflight, Conf)), + mqueue = emqx_mqueue:init(QueueOpts), + next_pkt_id = 1, + awaiting_rel = #{}, + max_subscriptions = maps:get(max_subscriptions, Conf), + max_awaiting_rel = maps:get(max_awaiting_rel, Conf), + upgrade_qos = maps:get(upgrade_qos, Conf), + retry_interval = maps:get(retry_interval, Conf), + await_rel_timeout = maps:get(await_rel_timeout, Conf) + }. + +get_mqueue_conf(Zone) -> + #{ + max_len => get_mqtt_conf(Zone, max_mqueue_len, 1000), + store_qos0 => get_mqtt_conf(Zone, mqueue_store_qos0), + priorities => get_mqtt_conf(Zone, mqueue_priorities), + default_priority => get_mqtt_conf(Zone, mqueue_default_priority) + }. + +get_mqtt_conf(Zone, Key) -> + emqx_config:get_zone_conf(Zone, [mqtt, Key]). + +get_mqtt_conf(Zone, Key, Default) -> + emqx_config:get_zone_conf(Zone, [mqtt, Key], Default). + +-spec destroy(session() | clientinfo()) -> ok. +destroy(_Session) -> + %% NOTE + %% This is a stub. This session impl has no backing store, thus always `ok`. + ok. + +%%-------------------------------------------------------------------- +%% Open a (possibly existing) Session +%%-------------------------------------------------------------------- + +-spec open(clientinfo(), conninfo()) -> + {_IsPresent :: true, session(), replayctx()} | _IsPresent :: false. +open(ClientInfo = #{clientid := ClientId}, _ConnInfo) -> + case emqx_cm:takeover_session_begin(ClientId) of + {ok, SessionRemote, TakeoverState} -> + Session = resume(ClientInfo, SessionRemote), + case emqx_cm:takeover_session_end(TakeoverState) of + {ok, Pendings} -> + clean_session(ClientInfo, Session, Pendings); + {error, _} -> + % TODO log error? + false + end; + none -> + false + end. + +clean_session(ClientInfo, Session = #session{mqueue = Q}, Pendings) -> + Q1 = emqx_mqueue:filter(fun emqx_session:should_keep/1, Q), + Session1 = Session#session{mqueue = Q1}, + Pendings1 = emqx_session:enrich_delivers(ClientInfo, Pendings, Session), + Pendings2 = lists:filter(fun emqx_session:should_keep/1, Pendings1), + {true, Session1, Pendings2}. + +%%-------------------------------------------------------------------- +%% Info, Stats +%%-------------------------------------------------------------------- + +%% @doc Get infos of the session. +info(Keys, Session) when is_list(Keys) -> + [{Key, info(Key, Session)} || Key <- Keys]; +info(id, #session{id = Id}) -> + Id; +info(clientid, #session{clientid = ClientId}) -> + ClientId; +info(created_at, #session{created_at = CreatedAt}) -> + CreatedAt; +info(is_persistent, #session{is_persistent = IsPersistent}) -> + IsPersistent; +info(subscriptions, #session{subscriptions = Subs}) -> + Subs; +info(subscriptions_cnt, #session{subscriptions = Subs}) -> + maps:size(Subs); +info(subscriptions_max, #session{max_subscriptions = MaxSubs}) -> + MaxSubs; +info(upgrade_qos, #session{upgrade_qos = UpgradeQoS}) -> + UpgradeQoS; +info(inflight, #session{inflight = Inflight}) -> + Inflight; +info(inflight_cnt, #session{inflight = Inflight}) -> + emqx_inflight:size(Inflight); +info(inflight_max, #session{inflight = Inflight}) -> + emqx_inflight:max_size(Inflight); +info(retry_interval, #session{retry_interval = Interval}) -> + Interval; +info(mqueue, #session{mqueue = MQueue}) -> + MQueue; +info(mqueue_len, #session{mqueue = MQueue}) -> + emqx_mqueue:len(MQueue); +info(mqueue_max, #session{mqueue = MQueue}) -> + emqx_mqueue:max_len(MQueue); +info(mqueue_dropped, #session{mqueue = MQueue}) -> + emqx_mqueue:dropped(MQueue); +info(next_pkt_id, #session{next_pkt_id = PacketId}) -> + PacketId; +info(awaiting_rel, #session{awaiting_rel = AwaitingRel}) -> + AwaitingRel; +info(awaiting_rel_cnt, #session{awaiting_rel = AwaitingRel}) -> + maps:size(AwaitingRel); +info(awaiting_rel_max, #session{max_awaiting_rel = Max}) -> + Max; +info(await_rel_timeout, #session{await_rel_timeout = Timeout}) -> + Timeout. + +%% @doc Get stats of the session. +-spec stats(session()) -> emqx_types:stats(). +stats(Session) -> info(?STATS_KEYS, Session). + +%%-------------------------------------------------------------------- +%% Client -> Broker: SUBSCRIBE / UNSUBSCRIBE +%%-------------------------------------------------------------------- + +-spec subscribe(emqx_types:topic(), emqx_types:subopts(), session()) -> + {ok, session()} | {error, emqx_types:reason_code()}. +subscribe( + TopicFilter, + SubOpts, + Session = #session{clientid = ClientId, subscriptions = Subs} +) -> + IsNew = not maps:is_key(TopicFilter, Subs), + case IsNew andalso is_subscriptions_full(Session) of + false -> + ok = emqx_broker:subscribe(TopicFilter, ClientId, SubOpts), + Session1 = Session#session{subscriptions = maps:put(TopicFilter, SubOpts, Subs)}, + {ok, Session1}; + true -> + {error, ?RC_QUOTA_EXCEEDED} + end. + +is_subscriptions_full(#session{max_subscriptions = infinity}) -> + false; +is_subscriptions_full(#session{ + subscriptions = Subs, + max_subscriptions = MaxLimit +}) -> + maps:size(Subs) >= MaxLimit. + +-spec unsubscribe(emqx_types:topic(), session()) -> + {ok, session(), emqx_types:subopts()} | {error, emqx_types:reason_code()}. +unsubscribe( + TopicFilter, + Session = #session{subscriptions = Subs} +) -> + case maps:find(TopicFilter, Subs) of + {ok, SubOpts} -> + ok = emqx_broker:unsubscribe(TopicFilter), + {ok, Session#session{subscriptions = maps:remove(TopicFilter, Subs)}, SubOpts}; + error -> + {error, ?RC_NO_SUBSCRIPTION_EXISTED} + end. + +-spec get_subscription(emqx_types:topic(), session()) -> + emqx_types:subopts() | undefined. +get_subscription(Topic, #session{subscriptions = Subs}) -> + maps:get(Topic, Subs, undefined). + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBLISH +%%-------------------------------------------------------------------- + +-spec publish(emqx_types:packet_id(), emqx_types:message(), session()) -> + {ok, emqx_types:publish_result(), session()} + | {error, emqx_types:reason_code()}. +publish( + PacketId, + Msg = #message{qos = ?QOS_2, timestamp = Ts}, + Session = #session{awaiting_rel = AwaitingRel} +) -> + case is_awaiting_full(Session) of + false -> + case maps:is_key(PacketId, AwaitingRel) of + false -> + Results = emqx_broker:publish(Msg), + AwaitingRel1 = maps:put(PacketId, Ts, AwaitingRel), + {ok, Results, Session#session{awaiting_rel = AwaitingRel1}}; + true -> + {error, ?RC_PACKET_IDENTIFIER_IN_USE} + end; + true -> + {error, ?RC_RECEIVE_MAXIMUM_EXCEEDED} + end; +%% Publish QoS0/1 directly +publish(_PacketId, Msg, Session) -> + {ok, emqx_broker:publish(Msg), [], Session}. + +is_awaiting_full(#session{max_awaiting_rel = infinity}) -> + false; +is_awaiting_full(#session{ + awaiting_rel = AwaitingRel, + max_awaiting_rel = MaxLimit +}) -> + maps:size(AwaitingRel) >= MaxLimit. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBACK +%%-------------------------------------------------------------------- + +-spec puback(clientinfo(), emqx_types:packet_id(), session()) -> + {ok, emqx_types:message(), replies(), session()} + | {error, emqx_types:reason_code()}. +puback(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, #inflight_data{phase = wait_ack, message = Msg}} -> + Inflight1 = emqx_inflight:delete(PacketId, Inflight), + Session1 = Session#session{inflight = Inflight1}, + {ok, Replies, Session2} = dequeue(ClientInfo, Session1), + {ok, Msg, Replies, Session2}; + {value, _} -> + {error, ?RC_PACKET_IDENTIFIER_IN_USE}; + none -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} + end. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBREC +%%-------------------------------------------------------------------- + +-spec pubrec(emqx_types:packet_id(), session()) -> + {ok, emqx_types:message(), session()} + | {error, emqx_types:reason_code()}. +pubrec(PacketId, Session = #session{inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, #inflight_data{phase = wait_ack, message = Msg} = Data} -> + Update = Data#inflight_data{phase = wait_comp}, + Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), + {ok, Msg, Session#session{inflight = Inflight1}}; + {value, _} -> + {error, ?RC_PACKET_IDENTIFIER_IN_USE}; + none -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} + end. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBREL +%%-------------------------------------------------------------------- + +-spec pubrel(emqx_types:packet_id(), session()) -> + {ok, session()} + | {error, emqx_types:reason_code()}. +pubrel(PacketId, Session = #session{awaiting_rel = AwaitingRel}) -> + case maps:take(PacketId, AwaitingRel) of + {_Ts, AwaitingRel1} -> + NSession = Session#session{awaiting_rel = AwaitingRel1}, + {ok, NSession}; + error -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} + end. + +%%-------------------------------------------------------------------- +%% Client -> Broker: PUBCOMP +%%-------------------------------------------------------------------- + +-spec pubcomp(clientinfo(), emqx_types:packet_id(), session()) -> + {ok, emqx_types:message(), replies(), session()} + | {error, emqx_types:reason_code()}. +pubcomp(ClientInfo, PacketId, Session = #session{inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, #inflight_data{phase = wait_comp, message = Msg}} -> + Inflight1 = emqx_inflight:delete(PacketId, Inflight), + Session1 = Session#session{inflight = Inflight1}, + {ok, Replies, Session2} = dequeue(ClientInfo, Session1), + {ok, Msg, Replies, Session2}; + {value, _Other} -> + {error, ?RC_PACKET_IDENTIFIER_IN_USE}; + none -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} + end. + +%%-------------------------------------------------------------------- +%% Dequeue Msgs +%%-------------------------------------------------------------------- + +dequeue(ClientInfo, Session = #session{inflight = Inflight, mqueue = Q}) -> + case emqx_mqueue:is_empty(Q) of + true -> + {ok, [], Session}; + false -> + {Msgs, Q1} = dequeue(ClientInfo, batch_n(Inflight), [], Q), + do_deliver(ClientInfo, Msgs, [], Session#session{mqueue = Q1}) + end. + +dequeue(_ClientInfo, 0, Msgs, Q) -> + {lists:reverse(Msgs), Q}; +dequeue(ClientInfo, Cnt, Msgs, Q) -> + case emqx_mqueue:out(Q) of + {empty, _Q} -> + dequeue(ClientInfo, 0, Msgs, Q); + {{value, Msg}, Q1} -> + case emqx_message:is_expired(Msg) of + true -> + _ = emqx_session_events:handle_event(ClientInfo, {expired, Msg}), + dequeue(ClientInfo, Cnt, Msgs, Q1); + false -> + dequeue(ClientInfo, acc_cnt(Msg, Cnt), [Msg | Msgs], Q1) + end + end. + +acc_cnt(#message{qos = ?QOS_0}, Cnt) -> Cnt; +acc_cnt(_Msg, Cnt) -> Cnt - 1. + +%%-------------------------------------------------------------------- +%% Broker -> Client: Deliver +%%-------------------------------------------------------------------- + +-spec deliver(clientinfo(), [emqx_types:deliver()], session()) -> + {ok, replies(), session()}. +deliver(ClientInfo, Msgs, Session) -> + do_deliver(ClientInfo, Msgs, [], Session). + +do_deliver(_ClientInfo, [], Publishes, Session) -> + {ok, lists:reverse(Publishes), Session}; +do_deliver(ClientInfo, [Msg | More], Acc, Session) -> + case deliver_msg(ClientInfo, Msg, Session) of + {ok, [], Session1} -> + do_deliver(ClientInfo, More, Acc, Session1); + {ok, [Publish], Session1} -> + do_deliver(ClientInfo, More, [Publish | Acc], Session1) + end. + +deliver_msg(_ClientInfo, Msg = #message{qos = ?QOS_0}, Session) -> + {ok, [{undefined, maybe_ack(Msg)}], Session}; +deliver_msg( + ClientInfo, + Msg = #message{qos = QoS}, + Session = #session{next_pkt_id = PacketId, inflight = Inflight} +) when + QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 +-> + case emqx_inflight:is_full(Inflight) of + true -> + Session1 = + case maybe_nack(Msg) of + true -> Session; + false -> enqueue_msg(ClientInfo, Msg, Session) + end, + {ok, [], Session1}; + false -> + %% Note that we publish message without shared ack header + %% But add to inflight with ack headers + %% This ack header is required for redispatch-on-terminate feature to work + Publish = {PacketId, maybe_ack(Msg)}, + MarkedMsg = mark_begin_deliver(Msg), + Inflight1 = emqx_inflight:insert(PacketId, with_ts(MarkedMsg), Inflight), + {ok, [Publish], next_pkt_id(Session#session{inflight = Inflight1})} + end. + +-spec enqueue(clientinfo(), [emqx_types:message()], session()) -> + session(). +enqueue(ClientInfo, Msgs, Session) when is_list(Msgs) -> + lists:foldl( + fun(Msg, Session0) -> enqueue_msg(ClientInfo, Msg, Session0) end, + Session, + Msgs + ). + +enqueue_msg(ClientInfo, #message{qos = QOS} = Msg, Session = #session{mqueue = Q}) -> + {Dropped, NQ} = emqx_mqueue:in(Msg, Q), + case Dropped of + undefined -> + Session#session{mqueue = NQ}; + _Msg -> + NQInfo = emqx_mqueue:info(NQ), + Reason = + case NQInfo of + #{store_qos0 := false} when QOS =:= ?QOS_0 -> qos0_msg; + _ -> queue_full + end, + _ = emqx_session_events:handle_event( + ClientInfo, + {dropped, Dropped, #{reason => Reason, logctx => #{queue => NQInfo}}} + ), + Session + end. + +maybe_ack(Msg) -> + emqx_shared_sub:maybe_ack(Msg). + +maybe_nack(Msg) -> + emqx_shared_sub:maybe_nack_dropped(Msg). + +mark_begin_deliver(Msg) -> + emqx_message:set_header(deliver_begin_at, erlang:system_time(millisecond), Msg). + +%%-------------------------------------------------------------------- +%% Timeouts +%%-------------------------------------------------------------------- + +%% @doc Handle timeout events +-spec handle_timeout(clientinfo(), emqx_session:common_timer_name(), session()) -> + {ok, replies(), session()} | {ok, replies(), timeout(), session()}. +handle_timeout(ClientInfo, retry_delivery, Session) -> + retry(ClientInfo, Session); +handle_timeout(ClientInfo, expire_awaiting_rel, Session) -> + expire(ClientInfo, Session). + +%%-------------------------------------------------------------------- +%% Retry Delivery +%%-------------------------------------------------------------------- + +-spec retry(clientinfo(), session()) -> + {ok, replies(), session()}. +retry(ClientInfo, Session = #session{inflight = Inflight}) -> + case emqx_inflight:is_empty(Inflight) of + true -> + {ok, [], Session}; + false -> + Now = erlang:system_time(millisecond), + retry_delivery( + ClientInfo, + emqx_inflight:to_list(fun sort_fun/2, Inflight), + [], + Now, + Session + ) + end. + +retry_delivery(_ClientInfo, [], Acc, _, Session = #session{retry_interval = Interval}) -> + {ok, lists:reverse(Acc), Interval, Session}; +retry_delivery( + ClientInfo, + [{PacketId, #inflight_data{timestamp = Ts} = Data} | More], + Acc, + Now, + Session = #session{retry_interval = Interval, inflight = Inflight} +) -> + case (Age = age(Now, Ts)) >= Interval of + true -> + {Acc1, Inflight1} = do_retry_delivery(ClientInfo, PacketId, Data, Now, Acc, Inflight), + retry_delivery(ClientInfo, More, Acc1, Now, Session#session{inflight = Inflight1}); + false -> + {ok, lists:reverse(Acc), Interval - max(0, Age), Session} + end. + +do_retry_delivery( + ClientInfo, + PacketId, + #inflight_data{phase = wait_ack, message = Msg} = Data, + Now, + Acc, + Inflight +) -> + case emqx_message:is_expired(Msg) of + true -> + _ = emqx_session_events:handle_event(ClientInfo, {expired, Msg}), + {Acc, emqx_inflight:delete(PacketId, Inflight)}; + false -> + Msg1 = emqx_message:set_flag(dup, true, Msg), + Update = Data#inflight_data{message = Msg1, timestamp = Now}, + Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), + {[{PacketId, Msg1} | Acc], Inflight1} + end; +do_retry_delivery(_ClientInfo, PacketId, Data, Now, Acc, Inflight) -> + Update = Data#inflight_data{timestamp = Now}, + Inflight1 = emqx_inflight:update(PacketId, Update, Inflight), + {[{pubrel, PacketId} | Acc], Inflight1}. + +%%-------------------------------------------------------------------- +%% Expire Awaiting Rel +%%-------------------------------------------------------------------- + +-spec expire(clientinfo(), session()) -> + {ok, replies(), session()}. +expire(ClientInfo, Session = #session{awaiting_rel = AwaitingRel}) -> + case maps:size(AwaitingRel) of + 0 -> + {ok, [], Session}; + _ -> + Now = erlang:system_time(millisecond), + expire_awaiting_rel(ClientInfo, Now, Session) + end. + +expire_awaiting_rel( + ClientInfo, + Now, + Session = #session{awaiting_rel = AwaitingRel, await_rel_timeout = Timeout} +) -> + NotExpired = fun(_PacketId, Ts) -> age(Now, Ts) < Timeout end, + AwaitingRel1 = maps:filter(NotExpired, AwaitingRel), + ExpiredCnt = maps:size(AwaitingRel) - maps:size(AwaitingRel1), + _ = emqx_session_events:handle_event(ClientInfo, {expired_rel, ExpiredCnt}), + Session1 = Session#session{awaiting_rel = AwaitingRel1}, + case maps:size(AwaitingRel1) of + 0 -> {ok, [], Session1}; + _ -> {ok, [], Timeout, Session1} + end. + +%%-------------------------------------------------------------------- +%% Takeover, Resume and Replay +%%-------------------------------------------------------------------- + +-spec takeover(session()) -> + ok. +takeover(#session{subscriptions = Subs}) -> + lists:foreach(fun emqx_broker:unsubscribe/1, maps:keys(Subs)). + +-spec resume(emqx_types:clientinfo(), session()) -> + session(). +resume(ClientInfo = #{clientid := ClientId}, Session = #session{subscriptions = Subs}) -> + ok = maps:foreach( + fun(TopicFilter, SubOpts) -> + ok = emqx_broker:subscribe(TopicFilter, ClientId, SubOpts) + end, + Subs + ), + ok = emqx_metrics:inc('session.resumed'), + ok = emqx_hooks:run('session.resumed', [ClientInfo, emqx_session:info(Session)]), + Session. + +-spec replay(emqx_types:clientinfo(), replayctx(), session()) -> + {ok, replies(), session()}. +replay(ClientInfo, Pendings, Session) -> + %% NOTE + %% Here, `Pendings` is a list messages that were pending delivery in the remote + %% session, see `clean_session/3`. It's a replay context that gets passed back + %% here after the remote session is taken over by `open/2`. When we have a set + %% of remote deliveries and a set of local deliveries, some publishes might actually + %% be in both sets, because there's a tiny amount of time when both remote and local + %% sessions were subscribed to the same set of topics simultaneously (i.e. after + %% local session calls `resume/2` but before remote session calls `takeover/1` + %% through `emqx_channel:handle_call({takeover, 'end'}, Channel)`). + %% We basically need to: + %% 1. Combine and deduplicate remote and local pending messages, so that no message + %% is delivered twice. + %% 2. Replay deliveries of the inflight messages, this time to the new channel. + %% 3. Deliver the combined pending messages, following the same logic as `deliver/3`. + PendingsAll = dedup(ClientInfo, Pendings, emqx_utils:drain_deliver(), Session), + {ok, PubsResendQueued, Session1} = replay(ClientInfo, Session), + {ok, PubsPending, Session2} = deliver(ClientInfo, PendingsAll, Session1), + {ok, append(PubsResendQueued, PubsPending), Session2}. + +-spec replay(emqx_types:clientinfo(), session()) -> + {ok, replies(), session()}. +replay(ClientInfo, Session) -> + PubsResend = lists:map( + fun + ({PacketId, #inflight_data{phase = wait_comp}}) -> + {pubrel, PacketId}; + ({PacketId, #inflight_data{message = Msg}}) -> + {PacketId, emqx_message:set_flag(dup, true, Msg)} + end, + emqx_inflight:to_list(Session#session.inflight) + ), + {ok, More, Session1} = dequeue(ClientInfo, Session), + {ok, append(PubsResend, More), Session1}. + +-spec dedup(clientinfo(), [emqx_types:message()], [emqx_types:deliver()], session()) -> + [emqx_types:message()]. +dedup(ClientInfo, Pendings, DeliversLocal, Session) -> + PendingsLocal1 = emqx_session:enrich_delivers(ClientInfo, DeliversLocal, Session), + PendingsLocal2 = lists:filter( + fun(Msg) -> not lists:keymember(Msg#message.id, #message.id, Pendings) end, + PendingsLocal1 + ), + append(Pendings, PendingsLocal2). + +append(L1, []) -> L1; +append(L1, L2) -> L1 ++ L2. + +%%-------------------------------------------------------------------- + +-spec disconnect(session()) -> {idle, session()}. +disconnect(Session = #session{}) -> + % TODO: isolate expiry timer / timeout handling here? + {idle, Session}. + +-spec terminate(Reason :: term(), session()) -> ok. +terminate(Reason, Session) -> + maybe_redispatch_shared_messages(Reason, Session), + ok. + +maybe_redispatch_shared_messages(takenover, _Session) -> + ok; +maybe_redispatch_shared_messages(kicked, _Session) -> + ok; +maybe_redispatch_shared_messages(_Reason, Session) -> + redispatch_shared_messages(Session). + +redispatch_shared_messages(#session{inflight = Inflight, mqueue = Q}) -> + AllInflights = emqx_inflight:to_list(fun sort_fun/2, Inflight), + F = fun + ({_PacketId, #inflight_data{message = #message{qos = ?QOS_1} = Msg}}) -> + %% For QoS 2, here is what the spec says: + %% If the Client's Session terminates before the Client reconnects, + %% the Server MUST NOT send the Application Message to any other + %% subscribed Client [MQTT-4.8.2-5]. + {true, Msg}; + ({_PacketId, #inflight_data{}}) -> + false + end, + InflightList = lists:filtermap(F, AllInflights), + emqx_shared_sub:redispatch(InflightList ++ emqx_mqueue:to_list(Q)). + +%%-------------------------------------------------------------------- +%% Next Packet Id +%%-------------------------------------------------------------------- + +obtain_next_pkt_id(Session) -> + {Session#session.next_pkt_id, next_pkt_id(Session)}. + +next_pkt_id(Session = #session{next_pkt_id = ?MAX_PACKET_ID}) -> + Session#session{next_pkt_id = 1}; +next_pkt_id(Session = #session{next_pkt_id = Id}) -> + Session#session{next_pkt_id = Id + 1}. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +-compile({inline, [sort_fun/2, batch_n/1, with_ts/1, age/2]}). + +sort_fun({_, A}, {_, B}) -> + A#inflight_data.timestamp =< B#inflight_data.timestamp. + +batch_n(Inflight) -> + case emqx_inflight:max_size(Inflight) of + 0 -> ?DEFAULT_BATCH_N; + Sz -> Sz - emqx_inflight:size(Inflight) + end. + +with_ts(Msg) -> + #inflight_data{ + phase = wait_ack, + message = Msg, + timestamp = erlang:system_time(millisecond) + }. + +age(Now, Ts) -> Now - Ts. + +%%-------------------------------------------------------------------- +%% For CT tests +%%-------------------------------------------------------------------- + +set_field(Name, Value, Session) -> + Pos = emqx_utils:index_of(Name, record_info(fields, session)), + setelement(Pos + 1, Session, Value). diff --git a/apps/emqx/src/emqx_session_router.erl b/apps/emqx/src/emqx_session_router.erl deleted file mode 100644 index 25484bdf0..000000000 --- a/apps/emqx/src/emqx_session_router.erl +++ /dev/null @@ -1,306 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_session_router). - --behaviour(gen_server). - --include("emqx.hrl"). --include("logger.hrl"). --include("types.hrl"). --include("persistent_session/emqx_persistent_session.hrl"). - --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --export([ - create_init_tab/0, - create_router_tab/1, - start_link/2 -]). - -%% Route APIs --export([ - delete_routes/2, - do_add_route/2, - do_delete_route/2, - match_routes/1 -]). - --export([ - buffer/3, - pending/2, - resume_begin/2, - resume_end/2 -]). - --export([print_routes/1]). - -%% gen_server callbacks --export([ - init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - terminate/2, - code_change/3 -]). - --type dest() :: node() | {emqx_types:group(), node()}. - --define(ROUTE_RAM_TAB, emqx_session_route_ram). --define(ROUTE_DISC_TAB, emqx_session_route_disc). - --define(SESSION_INIT_TAB, session_init_tab). - -%%-------------------------------------------------------------------- -%% Mnesia bootstrap -%%-------------------------------------------------------------------- - -create_router_tab(disc) -> - create_table(?ROUTE_DISC_TAB, disc_copies); -create_router_tab(ram) -> - create_table(?ROUTE_RAM_TAB, ram_copies). - -create_table(Tab, Storage) -> - ok = mria:create_table(Tab, [ - {type, bag}, - {rlog_shard, ?ROUTE_SHARD}, - {storage, Storage}, - {record_name, route}, - {attributes, record_info(fields, route)}, - {storage_properties, [ - {ets, [ - {read_concurrency, true}, - {write_concurrency, true} - ]} - ]} - ]). - -%%-------------------------------------------------------------------- -%% Start a router -%%-------------------------------------------------------------------- - -create_init_tab() -> - emqx_utils_ets:new(?SESSION_INIT_TAB, [ - public, - {read_concurrency, true}, - {write_concurrency, true} - ]). - --spec start_link(atom(), pos_integer()) -> startlink_ret(). -start_link(Pool, Id) -> - gen_server:start_link( - {local, emqx_utils:proc_name(?MODULE, Id)}, - ?MODULE, - [Pool, Id], - [{hibernate_after, 1000}] - ). - -%%-------------------------------------------------------------------- -%% Route APIs -%%-------------------------------------------------------------------- - --spec do_add_route(emqx_types:topic(), dest()) -> ok | {error, term()}. -do_add_route(Topic, SessionID) when is_binary(Topic) -> - Route = #route{topic = Topic, dest = SessionID}, - case lists:member(Route, lookup_routes(Topic)) of - true -> - ok; - false -> - case emqx_topic:wildcard(Topic) of - true -> - Fun = fun emqx_router_utils:insert_session_trie_route/2, - emqx_router_utils:maybe_trans( - Fun, - [route_tab(), Route], - ?PERSISTENT_SESSION_SHARD - ); - false -> - emqx_router_utils:insert_direct_route(route_tab(), Route) - end - end. - -%% @doc Match routes --spec match_routes(emqx_types:topic()) -> [emqx_types:route()]. -match_routes(Topic) when is_binary(Topic) -> - case match_trie(Topic) of - [] -> lookup_routes(Topic); - Matched -> lists:append([lookup_routes(To) || To <- [Topic | Matched]]) - end. - -%% Optimize: routing table will be replicated to all router nodes. -match_trie(Topic) -> - case emqx_trie:empty_session() of - true -> []; - false -> emqx_trie:match_session(Topic) - end. - -%% Async -delete_routes(SessionID, Subscriptions) -> - cast(pick(SessionID), {delete_routes, SessionID, Subscriptions}). - --spec do_delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}. -do_delete_route(Topic, SessionID) -> - Route = #route{topic = Topic, dest = SessionID}, - case emqx_topic:wildcard(Topic) of - true -> - Fun = fun emqx_router_utils:delete_session_trie_route/2, - emqx_router_utils:maybe_trans(Fun, [route_tab(), Route], ?PERSISTENT_SESSION_SHARD); - false -> - emqx_router_utils:delete_direct_route(route_tab(), Route) - end. - -%% @doc Print routes to a topic --spec print_routes(emqx_types:topic()) -> ok. -print_routes(Topic) -> - lists:foreach( - fun(#route{topic = To, dest = SessionID}) -> - io:format("~s -> ~p~n", [To, SessionID]) - end, - match_routes(Topic) - ). - -%%-------------------------------------------------------------------- -%% Session APIs -%%-------------------------------------------------------------------- - -pending(SessionID, MarkerIDs) -> - call(pick(SessionID), {pending, SessionID, MarkerIDs}). - -buffer(SessionID, STopic, Msg) -> - case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of - undefined -> ok; - Worker -> emqx_session_router_worker:buffer(Worker, STopic, Msg) - end. - --spec resume_begin(pid(), binary()) -> [{node(), emqx_guid:guid()}]. -resume_begin(From, SessionID) when is_pid(From), is_binary(SessionID) -> - call(pick(SessionID), {resume_begin, From, SessionID}). - --spec resume_end(pid(), binary()) -> - {'ok', [emqx_types:message()]} | {'error', term()}. -resume_end(From, SessionID) when is_pid(From), is_binary(SessionID) -> - case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of - undefined -> - ?tp(ps_session_not_found, #{sid => SessionID}), - {error, not_found}; - Pid -> - Res = emqx_session_router_worker:resume_end(From, Pid, SessionID), - cast(pick(SessionID), {resume_end, SessionID, Pid}), - Res - end. - -%%-------------------------------------------------------------------- -%% Worker internals -%%-------------------------------------------------------------------- - -call(Router, Msg) -> - gen_server:call(Router, Msg, infinity). - -cast(Router, Msg) -> - gen_server:cast(Router, Msg). - -pick(#route{dest = SessionID}) -> - gproc_pool:pick_worker(session_router_pool, SessionID); -pick(SessionID) when is_binary(SessionID) -> - gproc_pool:pick_worker(session_router_pool, SessionID). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id]) -> - true = gproc_pool:connect_worker(Pool, {Pool, Id}), - {ok, #{pool => Pool, id => Id, pmon => emqx_pmon:new()}}. - -handle_call({resume_begin, RemotePid, SessionID}, _From, State) -> - case init_resume_worker(RemotePid, SessionID, State) of - error -> - {reply, error, State}; - {ok, Pid, State1} -> - ets:insert(?SESSION_INIT_TAB, {SessionID, Pid}), - MarkerID = emqx_persistent_session:mark_resume_begin(SessionID), - {reply, {ok, MarkerID}, State1} - end; -handle_call({pending, SessionID, MarkerIDs}, _From, State) -> - Res = emqx_persistent_session:pending_messages_in_db(SessionID, MarkerIDs), - {reply, Res, State}; -handle_call(Req, _From, State) -> - ?SLOG(error, #{msg => "unexpected_call", req => Req}), - {reply, ignored, State}. - -handle_cast({delete_routes, SessionID, Subscriptions}, State) -> - %% TODO: Make a batch for deleting all routes. - Fun = fun(Topic, _) -> do_delete_route(Topic, SessionID) end, - ok = maps:foreach(Fun, Subscriptions), - {noreply, State}; -handle_cast({resume_end, SessionID, Pid}, State) -> - case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of - undefined -> skip; - P when P =:= Pid -> ets:delete(?SESSION_INIT_TAB, SessionID); - P when is_pid(P) -> skip - end, - Pmon = emqx_pmon:demonitor(Pid, maps:get(pmon, State)), - _ = emqx_session_router_worker_sup:abort_worker(Pid), - {noreply, State#{pmon => Pmon}}; -handle_cast(Msg, State) -> - ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), - {noreply, State}. - -handle_info(Info, State) -> - ?SLOG(error, #{msg => "unexpected_info", info => Info}), - {noreply, State}. - -terminate(_Reason, #{pool := Pool, id := Id}) -> - gproc_pool:disconnect_worker(Pool, {Pool, Id}). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Resume worker. A process that buffers the persisted messages during -%% initialisation of a resuming session. -%%-------------------------------------------------------------------- - -init_resume_worker(RemotePid, SessionID, #{pmon := Pmon} = State) -> - case emqx_session_router_worker_sup:start_worker(SessionID, RemotePid) of - {error, What} -> - ?SLOG(error, #{msg => "failed_to_start_resume_worker", reason => What}), - error; - {ok, Pid} -> - Pmon1 = emqx_pmon:monitor(Pid, Pmon), - case emqx_utils_ets:lookup_value(?SESSION_INIT_TAB, SessionID) of - undefined -> - {ok, Pid, State#{pmon => Pmon1}}; - {_, OldPid} -> - Pmon2 = emqx_pmon:demonitor(OldPid, Pmon1), - _ = emqx_session_router_worker_sup:abort_worker(OldPid), - {ok, Pid, State#{pmon => Pmon2}} - end - end. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -lookup_routes(Topic) -> - ets:lookup(route_tab(), Topic). - -route_tab() -> - case emqx_persistent_session:storage_type() of - disc -> ?ROUTE_DISC_TAB; - ram -> ?ROUTE_RAM_TAB - end. diff --git a/apps/emqx/src/emqx_sup.erl b/apps/emqx/src/emqx_sup.erl index 1893dba86..65742a234 100644 --- a/apps/emqx/src/emqx_sup.erl +++ b/apps/emqx/src/emqx_sup.erl @@ -67,13 +67,11 @@ init([]) -> KernelSup = child_spec(emqx_kernel_sup, supervisor), RouterSup = child_spec(emqx_router_sup, supervisor), BrokerSup = child_spec(emqx_broker_sup, supervisor), - SessionSup = child_spec(emqx_persistent_session_sup, supervisor), CMSup = child_spec(emqx_cm_sup, supervisor), SysSup = child_spec(emqx_sys_sup, supervisor), Limiter = child_spec(emqx_limiter_sup, supervisor), Children = [KernelSup] ++ - [SessionSup || emqx_persistent_session:is_store_enabled()] ++ [RouterSup || emqx_boot:is_enabled(broker)] ++ [BrokerSup || emqx_boot:is_enabled(broker)] ++ [CMSup || emqx_boot:is_enabled(broker)] ++ diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index cc937f81c..504540cf6 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -244,6 +244,7 @@ -type publish_result() :: [ {node(), topic(), deliver_result()} | {share, topic(), deliver_result()} + | persisted ]. -type route() :: #route{}. -type route_entry() :: {topic(), node()} | {topic, group()}. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session.erl b/apps/emqx/src/persistent_session/emqx_persistent_session.erl deleted file mode 100644 index d85e13d67..000000000 --- a/apps/emqx/src/persistent_session/emqx_persistent_session.erl +++ /dev/null @@ -1,562 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_persistent_session). - --export([ - is_store_enabled/0, - init_db_backend/0, - storage_backend/0, - storage_type/0 -]). - --export([ - discard/2, - discard_if_present/1, - lookup/1, - persist/3, - persist_message/1, - pending/1, - pending/2, - resume/3 -]). - --export([ - add_subscription/3, - remove_subscription/3 -]). - --export([ - mark_as_delivered/2, - mark_resume_begin/1 -]). - --export([ - pending_messages_in_db/2, - delete_session_message/1, - gc_session_messages/1, - session_message_info/2 -]). - --export([ - delete_message/1, - first_message_id/0, - next_message_id/1 -]). - --export_type([sess_msg_key/0]). - --include("emqx.hrl"). --include("emqx_channel.hrl"). --include("emqx_persistent_session.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --compile({inline, [is_store_enabled/0]}). - -%% NOTE: Order is significant because of traversal order of the table. --define(MARKER, 3). --define(DELIVERED, 2). --define(UNDELIVERED, 1). --define(ABANDONED, 0). - --type bin_timestamp() :: <<_:64>>. --opaque sess_msg_key() :: - {emqx_guid:guid(), emqx_guid:guid(), emqx_types:topic(), ?UNDELIVERED | ?DELIVERED} - | {emqx_guid:guid(), emqx_guid:guid(), <<>>, ?MARKER} - | {emqx_guid:guid(), <<>>, bin_timestamp(), ?ABANDONED}. - --type gc_traverse_fun() :: fun(('delete' | 'marker' | 'abandoned', sess_msg_key()) -> 'ok'). - -%% EMQX configuration keys --define(conf_storage_backend, [persistent_session_store, backend, type]). - -%%-------------------------------------------------------------------- -%% Init -%%-------------------------------------------------------------------- - -init_db_backend() -> - case is_store_enabled() of - true -> - StorageType = storage_type(), - ok = emqx_trie:create_session_trie(StorageType), - ok = emqx_session_router:create_router_tab(StorageType), - case storage_backend() of - builtin -> - emqx_persistent_session_backend_builtin:create_tables(), - persistent_term:put(?db_backend_key, emqx_persistent_session_backend_builtin) - end, - ok; - false -> - persistent_term:put(?db_backend_key, emqx_persistent_session_backend_dummy), - ok - end. - -is_store_enabled() -> - emqx_config:get(?is_enabled_key). - --spec storage_backend() -> builtin. -storage_backend() -> - emqx_config:get(?conf_storage_backend). - -%%-------------------------------------------------------------------- -%% Session message ADT API -%%-------------------------------------------------------------------- - --spec session_message_info('timestamp' | 'session_id', sess_msg_key()) -> term(). -session_message_info(timestamp, {_, <<>>, <>, ?ABANDONED}) -> TS; -session_message_info(timestamp, {_, GUID, _, _}) -> emqx_guid:timestamp(GUID); -session_message_info(session_id, {SessionID, _, _, _}) -> SessionID. - -%%-------------------------------------------------------------------- -%% DB API -%%-------------------------------------------------------------------- - -first_message_id() -> - ?db_backend:first_message_id(). - -next_message_id(Key) -> - ?db_backend:next_message_id(Key). - -delete_message(Key) -> - ?db_backend:delete_message(Key). - -first_session_message() -> - ?db_backend:first_session_message(). - -next_session_message(Key) -> - ?db_backend:next_session_message(Key). - -delete_session_message(Key) -> - ?db_backend:delete_session_message(Key). - -put_session_store(#session_store{} = SS) -> - ?db_backend:put_session_store(SS). - -delete_session_store(ClientID) -> - ?db_backend:delete_session_store(ClientID). - -lookup_session_store(ClientID) -> - ?db_backend:lookup_session_store(ClientID). - -put_session_message({_, _, _, _} = Key) -> - ?db_backend:put_session_message(#session_msg{key = Key}). - -put_message(Msg) -> - ?db_backend:put_message(Msg). - -get_message(MsgId) -> - ?db_backend:get_message(MsgId). - -pending_messages_in_db(SessionID, MarkerIds) -> - ?db_backend:ro_transaction(pending_messages_fun(SessionID, MarkerIds)). - -%%-------------------------------------------------------------------- -%% Session API -%%-------------------------------------------------------------------- - -%% The timestamp (TS) is the last time a client interacted with the session, -%% or when the client disconnected. --spec persist( - emqx_types:clientinfo(), - emqx_types:conninfo(), - emqx_session:session() -) -> emqx_session:session(). - -persist(#{clientid := ClientID}, ConnInfo, Session) -> - case ClientID == undefined orelse not emqx_session:info(is_persistent, Session) of - true -> - Session; - false -> - SS = #session_store{ - client_id = ClientID, - expiry_interval = maps:get(expiry_interval, ConnInfo), - ts = timestamp_from_conninfo(ConnInfo), - session = Session - }, - case persistent_session_status(SS) of - not_persistent -> - Session; - expired -> - discard(ClientID, Session); - persistent -> - put_session_store(SS), - Session - end - end. - -timestamp_from_conninfo(ConnInfo) -> - case maps:get(disconnected_at, ConnInfo, undefined) of - undefined -> erlang:system_time(millisecond); - Disconnect -> Disconnect - end. - -lookup(ClientID) when is_binary(ClientID) -> - case is_store_enabled() of - false -> - none; - true -> - case lookup_session_store(ClientID) of - none -> - none; - {value, #session_store{session = S} = SS} -> - case persistent_session_status(SS) of - expired -> {expired, S}; - persistent -> {persistent, S} - end - end - end. - --spec discard_if_present(binary()) -> 'ok'. -discard_if_present(ClientID) -> - case lookup(ClientID) of - none -> - ok; - {Tag, Session} when Tag =:= persistent; Tag =:= expired -> - _ = discard(ClientID, Session), - ok - end. - --spec discard(binary(), emqx_session:session()) -> emqx_session:session(). -discard(ClientID, Session) -> - discard_opt(is_store_enabled(), ClientID, Session). - -discard_opt(false, _ClientID, Session) -> - emqx_session:set_field(is_persistent, false, Session); -discard_opt(true, ClientID, Session) -> - delete_session_store(ClientID), - SessionID = emqx_session:info(id, Session), - put_session_message({SessionID, <<>>, <<(erlang:system_time(microsecond)):64>>, ?ABANDONED}), - Subscriptions = emqx_session:info(subscriptions, Session), - emqx_session_router:delete_routes(SessionID, Subscriptions), - emqx_session:set_field(is_persistent, false, Session). - --spec mark_resume_begin(emqx_session:session_id()) -> emqx_guid:guid(). -mark_resume_begin(SessionID) -> - MarkerID = emqx_guid:gen(), - put_session_message({SessionID, MarkerID, <<>>, ?MARKER}), - MarkerID. - -add_subscription(TopicFilter, SessionID, true = _IsPersistent) -> - case is_store_enabled() of - true -> emqx_session_router:do_add_route(TopicFilter, SessionID); - false -> ok - end; -add_subscription(_TopicFilter, _SessionID, false = _IsPersistent) -> - ok. - -remove_subscription(TopicFilter, SessionID, true = _IsPersistent) -> - case is_store_enabled() of - true -> emqx_session_router:do_delete_route(TopicFilter, SessionID); - false -> ok - end; -remove_subscription(_TopicFilter, _SessionID, false = _IsPersistent) -> - ok. - -%%-------------------------------------------------------------------- -%% Resuming from DB state -%%-------------------------------------------------------------------- - -%% Must be called inside a emqx_cm_locker transaction. --spec resume(emqx_types:clientinfo(), emqx_types:conninfo(), emqx_session:session()) -> - {emqx_session:session(), [emqx_types:deliver()]}. -resume(ClientInfo, ConnInfo, Session) -> - SessionID = emqx_session:info(id, Session), - ?tp(ps_resuming, #{from => db, sid => SessionID}), - - %% NOTE: Order is important! - - %% 1. Get pending messages from DB. - ?tp(ps_initial_pendings, #{sid => SessionID}), - Pendings1 = pending(SessionID), - ?tp(ps_got_initial_pendings, #{ - sid => SessionID, - msgs => Pendings1 - }), - - %% 2. Enqueue messages to mimic that the process was alive - %% when the messages were delivered. - ?tp(ps_persist_pendings, #{sid => SessionID}), - Session1 = emqx_session:enqueue(ClientInfo, Pendings1, Session), - Session2 = persist(ClientInfo, ConnInfo, Session1), - mark_as_delivered(SessionID, Pendings1), - ?tp(ps_persist_pendings_msgs, #{ - msgs => Pendings1, - sid => SessionID - }), - - %% 3. Notify writers that we are resuming. - %% They will buffer new messages. - ?tp(ps_notify_writers, #{sid => SessionID}), - Nodes = mria:running_nodes(), - NodeMarkers = resume_begin(Nodes, SessionID), - ?tp(ps_node_markers, #{sid => SessionID, markers => NodeMarkers}), - - %% 4. Subscribe to topics. - ?tp(ps_resume_session, #{sid => SessionID}), - ok = emqx_session:resume(ClientInfo, Session2), - - %% 5. Get pending messages from DB until we find all markers. - ?tp(ps_marker_pendings, #{sid => SessionID}), - MarkerIDs = [Marker || {_, Marker} <- NodeMarkers], - Pendings2 = pending(SessionID, MarkerIDs), - ?tp(ps_marker_pendings_msgs, #{ - sid => SessionID, - msgs => Pendings2 - }), - - %% 6. Get pending messages from writers. - ?tp(ps_resume_end, #{sid => SessionID}), - WriterPendings = resume_end(Nodes, SessionID), - ?tp(ps_writer_pendings, #{ - msgs => WriterPendings, - sid => SessionID - }), - - %% 7. Drain the inbox and usort the messages - %% with the pending messages. (Should be done by caller.) - {Session2, Pendings2 ++ WriterPendings}. - -resume_begin(Nodes, SessionID) -> - Res = emqx_persistent_session_proto_v1:resume_begin(Nodes, self(), SessionID), - [{Node, Marker} || {{ok, {ok, Marker}}, Node} <- lists:zip(Res, Nodes)]. - -resume_end(Nodes, SessionID) -> - Res = emqx_persistent_session_proto_v1:resume_end(Nodes, self(), SessionID), - ?tp(ps_erpc_multical_result, #{res => Res, sid => SessionID}), - %% TODO: Should handle the errors - [ - {deliver, STopic, M} - || {ok, {ok, Messages}} <- Res, - {{M, STopic}} <- Messages - ]. - -%%-------------------------------------------------------------------- -%% Messages API -%%-------------------------------------------------------------------- - -persist_message(Msg) -> - case is_store_enabled() of - true -> do_persist_message(Msg); - false -> ok - end. - -do_persist_message(Msg) -> - case emqx_message:get_flag(dup, Msg) orelse emqx_message:is_sys(Msg) of - true -> - ok; - false -> - case emqx_session_router:match_routes(emqx_message:topic(Msg)) of - [] -> - ok; - Routes -> - put_message(Msg), - MsgId = emqx_message:id(Msg), - persist_message_routes(Routes, MsgId, Msg) - end - end. - -persist_message_routes([#route{dest = SessionID, topic = STopic} | Left], MsgId, Msg) -> - ?tp(ps_persist_msg, #{sid => SessionID, payload => emqx_message:payload(Msg)}), - put_session_message({SessionID, MsgId, STopic, ?UNDELIVERED}), - emqx_session_router:buffer(SessionID, STopic, Msg), - persist_message_routes(Left, MsgId, Msg); -persist_message_routes([], _MsgId, _Msg) -> - ok. - -mark_as_delivered(SessionID, List) -> - case is_store_enabled() of - true -> do_mark_as_delivered(SessionID, List); - false -> ok - end. - -do_mark_as_delivered(SessionID, [{deliver, STopic, Msg} | Left]) -> - MsgID = emqx_message:id(Msg), - case next_session_message({SessionID, MsgID, STopic, ?ABANDONED}) of - {SessionID, MsgID, STopic, ?UNDELIVERED} = Key -> - %% We can safely delete this entry - %% instead of marking it as delivered. - delete_session_message(Key); - _ -> - put_session_message({SessionID, MsgID, STopic, ?DELIVERED}) - end, - do_mark_as_delivered(SessionID, Left); -do_mark_as_delivered(_SessionID, []) -> - ok. - --spec pending(emqx_session:session_id()) -> - [{emqx_types:message(), STopic :: binary()}]. -pending(SessionID) -> - pending_messages_in_db(SessionID, []). - --spec pending(emqx_session:session_id(), MarkerIDs :: [emqx_guid:guid()]) -> - [{emqx_types:message(), STopic :: binary()}]. -pending(SessionID, MarkerIds) -> - %% TODO: Handle lost MarkerIDs - case emqx_session_router:pending(SessionID, MarkerIds) of - incomplete -> - timer:sleep(10), - pending(SessionID, MarkerIds); - Delivers -> - Delivers - end. - -%%-------------------------------------------------------------------- -%% Session internal functions -%%-------------------------------------------------------------------- - -%% @private [MQTT-3.1.2-23] -persistent_session_status(#session_store{expiry_interval = 0}) -> - not_persistent; -persistent_session_status(#session_store{expiry_interval = ?EXPIRE_INTERVAL_INFINITE}) -> - persistent; -persistent_session_status(#session_store{expiry_interval = E, ts = TS}) -> - case E + TS > erlang:system_time(millisecond) of - true -> persistent; - false -> expired - end. - -%%-------------------------------------------------------------------- -%% Pending messages internal functions -%%-------------------------------------------------------------------- - -pending_messages_fun(SessionID, MarkerIds) -> - fun() -> - case pending_messages({SessionID, <<>>, <<>>, ?DELIVERED}, [], MarkerIds) of - {Pending, []} -> read_pending_msgs(Pending, []); - {_Pending, [_ | _]} -> incomplete - end - end. - -read_pending_msgs([{MsgId, STopic} | Left], Acc) -> - Acc1 = - try - [{deliver, STopic, get_message(MsgId)} | Acc] - catch - error:{msg_not_found, _} -> - HighwaterMark = - erlang:system_time(microsecond) - - emqx_config:get(?msg_retain) * 1000, - case emqx_guid:timestamp(MsgId) < HighwaterMark of - %% Probably cleaned by GC - true -> Acc; - false -> error({msg_not_found, MsgId}) - end - end, - read_pending_msgs(Left, Acc1); -read_pending_msgs([], Acc) -> - lists:reverse(Acc). - -%% The keys are ordered by -%% {session_id(), <<>>, bin_timestamp(), ?ABANDONED} For abandoned sessions (clean started or expired). -%% {session_id(), emqx_guid:guid(), STopic :: binary(), ?DELIVERED | ?UNDELIVERED | ?MARKER} -%% where -%% <<>> < emqx_guid:guid() -%% <<>> < bin_timestamp() -%% emqx_guid:guid() is ordered in ts() and by node() -%% ?ABANDONED < ?UNDELIVERED < ?DELIVERED < ?MARKER -%% -%% We traverse the table until we reach another session. -%% TODO: Garbage collect the delivered messages. -pending_messages({SessionID, PrevMsgId, PrevSTopic, PrevTag} = PrevKey, Acc, MarkerIds) -> - case next_session_message(PrevKey) of - {S, <<>>, _TS, ?ABANDONED} when S =:= SessionID -> - {[], []}; - {S, MsgId, <<>>, ?MARKER} = Key when S =:= SessionID -> - MarkerIds1 = MarkerIds -- [MsgId], - case PrevTag =:= ?UNDELIVERED of - false -> pending_messages(Key, Acc, MarkerIds1); - true -> pending_messages(Key, [{PrevMsgId, PrevSTopic} | Acc], MarkerIds1) - end; - {S, MsgId, STopic, ?DELIVERED} = Key when - S =:= SessionID, - MsgId =:= PrevMsgId, - STopic =:= PrevSTopic - -> - pending_messages(Key, Acc, MarkerIds); - {S, _MsgId, _STopic, _Tag} = Key when S =:= SessionID -> - case PrevTag =:= ?UNDELIVERED of - false -> pending_messages(Key, Acc, MarkerIds); - true -> pending_messages(Key, [{PrevMsgId, PrevSTopic} | Acc], MarkerIds) - end; - %% Next session_id or '$end_of_table' - _What -> - case PrevTag =:= ?UNDELIVERED of - false -> {lists:reverse(Acc), MarkerIds}; - true -> {lists:reverse([{PrevMsgId, PrevSTopic} | Acc]), MarkerIds} - end - end. - -%%-------------------------------------------------------------------- -%% Garbage collection -%%-------------------------------------------------------------------- - --spec gc_session_messages(gc_traverse_fun()) -> 'ok'. -gc_session_messages(Fun) -> - gc_traverse(first_session_message(), <<>>, false, Fun). - -gc_traverse('$end_of_table', _SessionID, _Abandoned, _Fun) -> - ok; -gc_traverse({S, <<>>, _TS, ?ABANDONED} = Key, _SessionID, _Abandoned, Fun) -> - %% Only report the abandoned session if it has no messages. - %% We want to keep the abandoned marker to last to make the GC reentrant. - case next_session_message(Key) of - '$end_of_table' = NextKey -> - ok = Fun(abandoned, Key), - gc_traverse(NextKey, S, true, Fun); - {S2, _, _, _} = NextKey when S =:= S2 -> - gc_traverse(NextKey, S, true, Fun); - {_, _, _, _} = NextKey -> - ok = Fun(abandoned, Key), - gc_traverse(NextKey, S, true, Fun) - end; -gc_traverse({S, _MsgID, <<>>, ?MARKER} = Key, SessionID, Abandoned, Fun) -> - ok = Fun(marker, Key), - NewAbandoned = S =:= SessionID andalso Abandoned, - gc_traverse(next_session_message(Key), S, NewAbandoned, Fun); -gc_traverse({S, _MsgID, _STopic, _Tag} = Key, SessionID, Abandoned, Fun) when - Abandoned andalso - S =:= SessionID --> - %% Delete all messages from an abandoned session. - ok = Fun(delete, Key), - gc_traverse(next_session_message(Key), S, Abandoned, Fun); -gc_traverse({S, MsgID, STopic, ?UNDELIVERED} = Key, SessionID, Abandoned, Fun) -> - case next_session_message(Key) of - {S1, M, ST, ?DELIVERED} = NextKey when - S1 =:= S andalso - MsgID =:= M andalso - STopic =:= ST - -> - %% We have both markers for the same message/topic so it is safe to delete both. - ok = Fun(delete, Key), - ok = Fun(delete, NextKey), - gc_traverse(next_session_message(NextKey), S, Abandoned, Fun); - NextKey -> - %% Something else is here, so let's just loop. - NewAbandoned = S =:= SessionID andalso Abandoned, - gc_traverse(NextKey, SessionID, NewAbandoned, Fun) - end; -gc_traverse({S, _MsgID, _STopic, ?DELIVERED} = Key, SessionID, Abandoned, Fun) -> - %% We have a message that is marked as ?DELIVERED, but the ?UNDELIVERED is missing. - NewAbandoned = S =:= SessionID andalso Abandoned, - gc_traverse(next_session_message(Key), S, NewAbandoned, Fun). - --spec storage_type() -> ram | disc. -storage_type() -> - case emqx_config:get(?on_disc_key) of - true -> disc; - false -> ram - end. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session.hrl b/apps/emqx/src/persistent_session/emqx_persistent_session.hrl deleted file mode 100644 index 5476d8daf..000000000 --- a/apps/emqx/src/persistent_session/emqx_persistent_session.hrl +++ /dev/null @@ -1,41 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --define(PERSISTENT_SESSION_SHARD, emqx_persistent_session_shard). - --record(session_store, { - client_id :: binary(), - expiry_interval :: non_neg_integer(), - ts :: non_neg_integer(), - session :: emqx_session:session() -}). - --record(session_msg, { - key :: emqx_persistent_session:sess_msg_key(), - val = [] :: [] -}). - --define(cfg_root, persistent_session_store). --define(db_backend_key, [?cfg_root, db_backend]). --define(is_enabled_key, [?cfg_root, enabled]). --define(msg_retain, [?cfg_root, max_retain_undelivered]). --define(on_disc_key, [?cfg_root, on_disc]). - --define(SESSION_STORE, emqx_session_store). --define(SESS_MSG_TAB, emqx_session_msg). --define(MSG_TAB, emqx_persistent_msg). - --define(db_backend, (persistent_term:get(?db_backend_key))). diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl deleted file mode 100644 index 34305d7bc..000000000 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_builtin.erl +++ /dev/null @@ -1,157 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_persistent_session_backend_builtin). - --include("emqx.hrl"). --include_lib("typerefl/include/types.hrl"). --include("emqx_persistent_session.hrl"). - --export([ - create_tables/0, - first_message_id/0, - next_message_id/1, - delete_message/1, - first_session_message/0, - next_session_message/1, - delete_session_message/1, - put_session_store/1, - delete_session_store/1, - lookup_session_store/1, - put_session_message/1, - put_message/1, - get_message/1, - ro_transaction/1 -]). - --type mria_table_type() :: ram_copies | disc_copies | rocksdb_copies. - --define(IS_ETS(BACKEND), (BACKEND =:= ram_copies orelse BACKEND =:= disc_copies)). - -create_tables() -> - SessStoreBackend = table_type(session), - ok = mria:create_table(?SESSION_STORE, [ - {type, set}, - {rlog_shard, ?PERSISTENT_SESSION_SHARD}, - {storage, SessStoreBackend}, - {record_name, session_store}, - {attributes, record_info(fields, session_store)}, - {storage_properties, storage_properties(?SESSION_STORE, SessStoreBackend)} - ]), - - SessMsgBackend = table_type(session_messages), - ok = mria:create_table(?SESS_MSG_TAB, [ - {type, ordered_set}, - {rlog_shard, ?PERSISTENT_SESSION_SHARD}, - {storage, SessMsgBackend}, - {record_name, session_msg}, - {attributes, record_info(fields, session_msg)}, - {storage_properties, storage_properties(?SESS_MSG_TAB, SessMsgBackend)} - ]), - - MsgBackend = table_type(messages), - ok = mria:create_table(?MSG_TAB, [ - {type, ordered_set}, - {rlog_shard, ?PERSISTENT_SESSION_SHARD}, - {storage, MsgBackend}, - {record_name, message}, - {attributes, record_info(fields, message)}, - {storage_properties, storage_properties(?MSG_TAB, MsgBackend)} - ]). - -first_session_message() -> - mnesia:dirty_first(?SESS_MSG_TAB). - -next_session_message(Key) -> - mnesia:dirty_next(?SESS_MSG_TAB, Key). - -first_message_id() -> - mnesia:dirty_first(?MSG_TAB). - -next_message_id(Key) -> - mnesia:dirty_next(?MSG_TAB, Key). - -delete_message(Key) -> - mria:dirty_delete(?MSG_TAB, Key). - -delete_session_message(Key) -> - mria:dirty_delete(?SESS_MSG_TAB, Key). - -put_session_store(SS) -> - mria:dirty_write(?SESSION_STORE, SS). - -delete_session_store(ClientID) -> - mria:dirty_delete(?SESSION_STORE, ClientID). - -lookup_session_store(ClientID) -> - case mnesia:dirty_read(?SESSION_STORE, ClientID) of - [] -> none; - [SS] -> {value, SS} - end. - -put_session_message(SessMsg) -> - mria:dirty_write(?SESS_MSG_TAB, SessMsg). - -put_message(Msg) -> - mria:dirty_write(?MSG_TAB, Msg). - -get_message(MsgId) -> - case mnesia:read(?MSG_TAB, MsgId) of - [] -> error({msg_not_found, MsgId}); - [Msg] -> Msg - end. - -ro_transaction(Fun) -> - {atomic, Res} = mria:ro_transaction(?PERSISTENT_SESSION_SHARD, Fun), - Res. - --spec storage_properties(?SESSION_STORE | ?SESS_MSG_TAB | ?MSG_TAB, mria_table_type()) -> term(). -storage_properties(?SESSION_STORE, Backend) when ?IS_ETS(Backend) -> - [{ets, [{read_concurrency, true}]}]; -storage_properties(_, Backend) when ?IS_ETS(Backend) -> - [ - {ets, [ - {read_concurrency, true}, - {write_concurrency, true} - ]} - ]; -storage_properties(_, _) -> - []. - -%% Dialyzer sees the compiled literal in -%% `mria:rocksdb_backend_available/0' and complains about the -%% complementar match arm... --dialyzer({no_match, table_type/1}). --spec table_type(atom()) -> mria_table_type(). -table_type(Table) -> - DiscPersistence = emqx_config:get([?cfg_root, on_disc]), - RamCache = get_overlayed(Table, ram_cache), - RocksDBAvailable = mria:rocksdb_backend_available(), - case {DiscPersistence, RamCache, RocksDBAvailable} of - {true, true, _} -> - disc_copies; - {true, false, true} -> - rocksdb_copies; - {true, false, false} -> - disc_copies; - {false, _, _} -> - ram_copies - end. - --spec get_overlayed(atom(), on_disc | ram_cache) -> boolean(). -get_overlayed(Table, Suffix) -> - Default = emqx_config:get([?cfg_root, Suffix]), - emqx_config:get([?cfg_root, backend, Table, Suffix], Default). diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_dummy.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_backend_dummy.erl deleted file mode 100644 index 1b8beef33..000000000 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_backend_dummy.erl +++ /dev/null @@ -1,76 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_persistent_session_backend_dummy). - --include("emqx_persistent_session.hrl"). - --export([ - first_message_id/0, - next_message_id/1, - delete_message/1, - first_session_message/0, - next_session_message/1, - delete_session_message/1, - put_session_store/1, - delete_session_store/1, - lookup_session_store/1, - put_session_message/1, - put_message/1, - get_message/1, - ro_transaction/1 -]). - -first_message_id() -> - '$end_of_table'. - -next_message_id(_) -> - '$end_of_table'. - --spec delete_message(binary()) -> no_return(). -delete_message(_Key) -> - error(should_not_be_called). - -first_session_message() -> - '$end_of_table'. - -next_session_message(_Key) -> - '$end_of_table'. - -delete_session_message(_Key) -> - ok. - -put_session_store(#session_store{}) -> - ok. - -delete_session_store(_ClientID) -> - ok. - -lookup_session_store(_ClientID) -> - none. - -put_session_message({_, _, _, _}) -> - ok. - -put_message(_Msg) -> - ok. - --spec get_message(binary()) -> no_return(). -get_message(_MsgId) -> - error(should_not_be_called). - -ro_transaction(Fun) -> - Fun(). diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_gc.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_gc.erl deleted file mode 100644 index 4aa59cdef..000000000 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_gc.erl +++ /dev/null @@ -1,163 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_persistent_session_gc). - --behaviour(gen_server). - --include("emqx_persistent_session.hrl"). - -%% API --export([start_link/0]). - -%% gen_server callbacks --export([ - init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - terminate/2 -]). - --ifdef(TEST). --export([ - session_gc_worker/2, - message_gc_worker/0 -]). --endif. - --define(SERVER, ?MODULE). -%% TODO: Maybe these should be configurable? --define(MARKER_GRACE_PERIOD, 60000000). --define(ABANDONED_GRACE_PERIOD, 300000000). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - process_flag(trap_exit, true), - mria_rlog:ensure_shard(?PERSISTENT_SESSION_SHARD), - {ok, start_message_gc_timer(start_session_gc_timer(#{}))}. - -handle_call(_Request, _From, State) -> - Reply = ok, - {reply, Reply, State}. - -handle_cast(_Request, State) -> - {noreply, State}. - -handle_info({timeout, Ref, session_gc_timeout}, State) -> - State1 = session_gc_timeout(Ref, State), - {noreply, State1}; -handle_info({timeout, Ref, message_gc_timeout}, State) -> - State1 = message_gc_timeout(Ref, State), - {noreply, State1}; -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% Session messages GC -%%-------------------------------------------------------------------- - -start_session_gc_timer(State) -> - Interval = emqx_config:get([persistent_session_store, session_message_gc_interval]), - State#{session_gc_timer => erlang:start_timer(Interval, self(), session_gc_timeout)}. - -session_gc_timeout(Ref, #{session_gc_timer := R} = State) when R =:= Ref -> - %% Prevent overlapping processes. - GCPid = maps:get(session_gc_pid, State, undefined), - case GCPid =/= undefined andalso erlang:is_process_alive(GCPid) of - true -> - start_session_gc_timer(State); - false -> - start_session_gc_timer(State#{ - session_gc_pid => proc_lib:spawn_link(fun session_gc_worker/0) - }) - end; -session_gc_timeout(_Ref, State) -> - State. - -session_gc_worker() -> - ok = emqx_persistent_session:gc_session_messages(fun session_gc_worker/2). - -session_gc_worker(delete, Key) -> - emqx_persistent_session:delete_session_message(Key); -session_gc_worker(marker, Key) -> - TS = emqx_persistent_session:session_message_info(timestamp, Key), - case TS + ?MARKER_GRACE_PERIOD < erlang:system_time(microsecond) of - true -> emqx_persistent_session:delete_session_message(Key); - false -> ok - end; -session_gc_worker(abandoned, Key) -> - TS = emqx_persistent_session:session_message_info(timestamp, Key), - case TS + ?ABANDONED_GRACE_PERIOD < erlang:system_time(microsecond) of - true -> emqx_persistent_session:delete_session_message(Key); - false -> ok - end. - -%%-------------------------------------------------------------------- -%% Message GC -%% -------------------------------------------------------------------- -%% The message GC simply removes all messages older than the retain -%% period. A more exact GC would either involve treating the session -%% message table as root set, or some kind of reference counting. -%% We sacrifice space for simplicity at this point. -start_message_gc_timer(State) -> - Interval = emqx_config:get([persistent_session_store, session_message_gc_interval]), - State#{message_gc_timer => erlang:start_timer(Interval, self(), message_gc_timeout)}. - -message_gc_timeout(Ref, #{message_gc_timer := R} = State) when R =:= Ref -> - %% Prevent overlapping processes. - GCPid = maps:get(message_gc_pid, State, undefined), - case GCPid =/= undefined andalso erlang:is_process_alive(GCPid) of - true -> - start_message_gc_timer(State); - false -> - start_message_gc_timer(State#{ - message_gc_pid => proc_lib:spawn_link(fun message_gc_worker/0) - }) - end; -message_gc_timeout(_Ref, State) -> - State. - -message_gc_worker() -> - HighWaterMark = erlang:system_time(microsecond) - emqx_config:get(?msg_retain) * 1000, - message_gc_worker(emqx_persistent_session:first_message_id(), HighWaterMark). - -message_gc_worker('$end_of_table', _HighWaterMark) -> - ok; -message_gc_worker(MsgId, HighWaterMark) -> - case emqx_guid:timestamp(MsgId) < HighWaterMark of - true -> - emqx_persistent_session:delete_message(MsgId), - message_gc_worker(emqx_persistent_session:next_message_id(MsgId), HighWaterMark); - false -> - ok - end. diff --git a/apps/emqx/src/persistent_session/emqx_persistent_session_sup.erl b/apps/emqx/src/persistent_session/emqx_persistent_session_sup.erl deleted file mode 100644 index 3018df96a..000000000 --- a/apps/emqx/src/persistent_session/emqx_persistent_session_sup.erl +++ /dev/null @@ -1,69 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_persistent_session_sup). - --behaviour(supervisor). - --export([start_link/0]). - --export([init/1]). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - %% We want this supervisor to own the table for restarts - SessionTab = emqx_session_router:create_init_tab(), - - %% Resume worker sup - ResumeSup = #{ - id => router_worker_sup, - start => {emqx_session_router_worker_sup, start_link, [SessionTab]}, - restart => permanent, - shutdown => 2000, - type => supervisor, - modules => [emqx_session_router_worker_sup] - }, - - SessionRouterPool = emqx_pool_sup:spec( - session_router_pool, - [ - session_router_pool, - hash, - {emqx_session_router, start_link, []} - ] - ), - - GCWorker = child_spec(emqx_persistent_session_gc, worker), - - Spec = #{ - strategy => one_for_all, - intensity => 0, - period => 1 - }, - - {ok, {Spec, [ResumeSup, SessionRouterPool, GCWorker]}}. - -child_spec(Mod, worker) -> - #{ - id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 15000, - type => worker, - modules => [Mod] - }. diff --git a/apps/emqx/src/proto/emqx_cm_proto_v2.erl b/apps/emqx/src/proto/emqx_cm_proto_v2.erl index b1eebd2d4..29dec50cd 100644 --- a/apps/emqx/src/proto/emqx_cm_proto_v2.erl +++ b/apps/emqx/src/proto/emqx_cm_proto_v2.erl @@ -48,11 +48,13 @@ kickout_client(Node, ClientId) -> lookup_client(Node, Key) -> rpc:call(Node, emqx_cm, lookup_client, [Key]). --spec get_chan_stats(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:stats() | {badrpc, _}. +-spec get_chan_stats(emqx_types:clientid(), emqx_cm:chan_pid()) -> + emqx_types:stats() | undefined | {badrpc, _}. get_chan_stats(ClientId, ChanPid) -> rpc:call(node(ChanPid), emqx_cm, do_get_chan_stats, [ClientId, ChanPid], ?T_GET_INFO * 2). --spec get_chan_info(emqx_types:clientid(), emqx_cm:chan_pid()) -> emqx_types:infos() | {badrpc, _}. +-spec get_chan_info(emqx_types:clientid(), emqx_cm:chan_pid()) -> + emqx_types:infos() | undefined | {badrpc, _}. get_chan_info(ClientId, ChanPid) -> rpc:call(node(ChanPid), emqx_cm, do_get_chan_info, [ClientId, ChanPid], ?T_GET_INFO * 2). diff --git a/apps/emqx/src/proto/emqx_persistent_session_ds_proto_v1.erl b/apps/emqx/src/proto/emqx_persistent_session_ds_proto_v1.erl index b1926098d..47c9ed541 100644 --- a/apps/emqx/src/proto/emqx_persistent_session_ds_proto_v1.erl +++ b/apps/emqx/src/proto/emqx_persistent_session_ds_proto_v1.erl @@ -36,7 +36,7 @@ introduced_in() -> -spec open_iterator( [node()], - emqx_topic:words(), + emqx_types:words(), emqx_ds:time(), emqx_ds:iterator_id() ) -> diff --git a/apps/emqx/src/proto/emqx_persistent_session_proto_v1.erl b/apps/emqx/src/proto/emqx_persistent_session_proto_v1.erl deleted file mode 100644 index 875f19852..000000000 --- a/apps/emqx/src/proto/emqx_persistent_session_proto_v1.erl +++ /dev/null @@ -1,41 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_persistent_session_proto_v1). - --behaviour(emqx_bpapi). - --export([ - introduced_in/0, - resume_begin/3, - resume_end/3 -]). - --include("bpapi.hrl"). --include("emqx.hrl"). - -introduced_in() -> - "5.0.0". - --spec resume_begin([node()], pid(), binary()) -> - emqx_rpc:erpc_multicall([{node(), emqx_guid:guid()}]). -resume_begin(Nodes, Pid, SessionID) when is_pid(Pid), is_binary(SessionID) -> - erpc:multicall(Nodes, emqx_session_router, resume_begin, [Pid, SessionID]). - --spec resume_end([node()], pid(), binary()) -> - emqx_rpc:erpc_multicall({'ok', [emqx_types:message()]} | {'error', term()}). -resume_end(Nodes, Pid, SessionID) when is_pid(Pid), is_binary(SessionID) -> - erpc:multicall(Nodes, emqx_session_router, resume_end, [Pid, SessionID]). diff --git a/apps/emqx/test/emqx_bpapi_static_checks.erl b/apps/emqx/test/emqx_bpapi_static_checks.erl index 56baf05e8..b44e564c7 100644 --- a/apps/emqx/test/emqx_bpapi_static_checks.erl +++ b/apps/emqx/test/emqx_bpapi_static_checks.erl @@ -53,11 +53,13 @@ -define(IGNORED_MODULES, "emqx_rpc"). -define(FORCE_DELETED_MODULES, [ emqx_statsd, - emqx_statsd_proto_v1 + emqx_statsd_proto_v1, + emqx_persistent_session_proto_v1 ]). -define(FORCE_DELETED_APIS, [ {emqx_statsd, 1}, - {emqx_plugin_libs, 1} + {emqx_plugin_libs, 1}, + {emqx_persistent_session, 1} ]). %% List of known RPC backend modules: -define(RPC_MODULES, "gen_rpc, erpc, rpc, emqx_rpc"). diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 6520d820a..408ae0014 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -38,48 +38,30 @@ init_per_suite(Config) -> ok = meck:expect(emqx_cm, mark_channel_disconnected, fun(_) -> ok end), %% Broker Meck ok = meck:new(emqx_broker, [passthrough, no_history, no_link]), - %% Hooks Meck - ok = meck:new(emqx_hooks, [passthrough, no_history, no_link]), - ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end), - ok = meck:expect(emqx_hooks, run_fold, fun(_Hook, _Args, Acc) -> Acc end), %% Session Meck ok = meck:new(emqx_session, [passthrough, no_history, no_link]), - %% Metrics - ok = meck:new(emqx_metrics, [passthrough, no_history, no_link]), - ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), - ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end), %% Ban meck:new(emqx_banned, [passthrough, no_history, no_link]), ok = meck:expect(emqx_banned, check, fun(_ConnInfo) -> false end), - Config. + Apps = emqx_cth_suite:start( + [ + {emqx, #{ + override_env => [{boot_modules, [broker]}] + }} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{suite_apps, Apps} | Config]. -end_per_suite(_Config) -> +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(suite_apps, Config)), meck:unload([ - emqx_metrics, emqx_session, emqx_broker, - emqx_hooks, emqx_cm, emqx_banned ]). -init_per_testcase(_TestCase, Config) -> - %% Access Control Meck - ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), - ok = meck:expect( - emqx_access_control, - authenticate, - fun(_) -> {ok, #{is_superuser => false}} end - ), - ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end), - emqx_common_test_helpers:start_apps([]), - Config. - -end_per_testcase(_TestCase, Config) -> - meck:unload([emqx_access_control]), - emqx_common_test_helpers:stop_apps([]), - Config. - %%-------------------------------------------------------------------- %% Test cases for channel info/stats/caps %%-------------------------------------------------------------------- @@ -111,14 +93,7 @@ t_chan_caps(_) -> %% Test cases for channel handle_in %%-------------------------------------------------------------------- -t_handle_in_connect_packet_sucess(_) -> - ok = meck:expect( - emqx_cm, - open_session, - fun(true, _ClientInfo, _ConnInfo) -> - {ok, #{session => session(), present => false}} - end - ), +t_handle_in_connect_packet_success(_) -> IdleChannel = channel(#{conn_state => idle}), {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, _)}], Channel} = emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), IdleChannel), @@ -242,7 +217,6 @@ t_handle_in_qos2_publish(_) -> ?assertEqual(2, proplists:get_value(awaiting_rel_cnt, emqx_channel:stats(Channel2))). t_handle_in_qos2_publish_with_error_return(_) -> - ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), Session = session(#{max_awaiting_rel => 2, awaiting_rel => #{1 => 1}}), Channel = channel(#{conn_state => connected, session => Session}), @@ -268,7 +242,7 @@ t_handle_in_puback_ok(_) -> ok = meck:expect( emqx_session, puback, - fun(_, _PacketId, Session) -> {ok, Msg, Session} end + fun(_, _PacketId, Session) -> {ok, Msg, [], Session} end ), Channel = channel(#{conn_state => connected}), {ok, _NChannel} = emqx_channel:handle_in(?PUBACK_PACKET(1, ?RC_SUCCESS), Channel). @@ -379,7 +353,7 @@ t_handle_in_pubrel_not_found_error(_) -> emqx_channel:handle_in(?PUBREL_PACKET(1, ?RC_SUCCESS), channel()). t_handle_in_pubcomp_ok(_) -> - ok = meck:expect(emqx_session, pubcomp, fun(_, _, Session) -> {ok, Session} end), + ok = meck:expect(emqx_session, pubcomp, fun(_, _, Session) -> {ok, [], Session} end), {ok, _Channel} = emqx_channel:handle_in(?PUBCOMP_PACKET(1, ?RC_SUCCESS), channel()). % ?assertEqual(#{pubcomp_in => 1}, emqx_channel:info(pub_stats, Channel)). @@ -491,18 +465,7 @@ t_process_unsubscribe(_) -> t_quota_qos0(_) -> esockd_limiter:start_link(), add_bucket(), - Cnter = counters:new(1, []), ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, {ok, 4}}] end), - ok = meck:expect( - emqx_metrics, - inc, - fun('packets.publish.dropped') -> counters:add(Cnter, 1, 1) end - ), - ok = meck:expect( - emqx_metrics, - val, - fun('packets.publish.dropped') -> counters:get(Cnter, 1) end - ), Chann = channel(#{conn_state => connected, quota => quota()}), Pub = ?PUBLISH_PACKET(?QOS_0, <<"topic">>, undefined, <<"payload">>), @@ -515,8 +478,6 @@ t_quota_qos0(_) -> {ok, _} = emqx_channel:handle_in(Pub, Chann3), M1 = emqx_metrics:val('packets.publish.dropped') - 1, - ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), - ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end), del_bucket(), esockd_limiter:stop(). @@ -741,7 +702,7 @@ t_handle_call_takeover_begin(_) -> {reply, _Session, _Chan} = emqx_channel:handle_call({takeover, 'begin'}, channel()). t_handle_call_takeover_end(_) -> - ok = meck:expect(emqx_session, takeover, fun(_) -> ok end), + ok = meck:expect(emqx_broker, unsubscribe, fun(_) -> ok end), {shutdown, takenover, [], _, _Chan} = emqx_channel:handle_call({takeover, 'end'}, channel()). @@ -768,32 +729,24 @@ t_handle_info_sock_closed(_) -> %% Test cases for handle_timeout %%-------------------------------------------------------------------- -t_handle_timeout_emit_stats(_) -> - TRef = make_ref(), - ok = meck:expect(emqx_cm, set_chan_stats, fun(_, _) -> ok end), - Channel = emqx_channel:set_field(timers, #{stats_timer => TRef}, channel()), - {ok, _Chan} = emqx_channel:handle_timeout(TRef, {emit_stats, []}, Channel). - t_handle_timeout_keepalive(_) -> TRef = make_ref(), - Channel = emqx_channel:set_field(timers, #{alive_timer => TRef}, channel()), + Channel = emqx_channel:set_field(timers, #{keepalive => TRef}, channel()), {ok, _Chan} = emqx_channel:handle_timeout(make_ref(), {keepalive, 10}, Channel). t_handle_timeout_retry_delivery(_) -> TRef = make_ref(), - ok = meck:expect(emqx_session, retry, fun(_, Session) -> {ok, Session} end), - Channel = emqx_channel:set_field(timers, #{retry_timer => TRef}, channel()), + Channel = emqx_channel:set_field(timers, #{retry_delivery => TRef}, channel()), {ok, _Chan} = emqx_channel:handle_timeout(TRef, retry_delivery, Channel). t_handle_timeout_expire_awaiting_rel(_) -> TRef = make_ref(), - ok = meck:expect(emqx_session, expire, fun(_, _, Session) -> {ok, Session} end), - Channel = emqx_channel:set_field(timers, #{await_timer => TRef}, channel()), + Channel = emqx_channel:set_field(timers, #{expire_awaiting_rel => TRef}, channel()), {ok, _Chan} = emqx_channel:handle_timeout(TRef, expire_awaiting_rel, Channel). t_handle_timeout_expire_session(_) -> TRef = make_ref(), - Channel = emqx_channel:set_field(timers, #{expire_timer => TRef}, channel()), + Channel = emqx_channel:set_field(timers, #{expire_session => TRef}, channel()), {shutdown, expired, _Chan} = emqx_channel:handle_timeout(TRef, expire_session, Channel). t_handle_timeout_will_message(_) -> @@ -983,9 +936,14 @@ t_flapping_detect(_) -> {ok, #{session => session(), present => false}} end ), - ok = meck:expect(emqx_access_control, authenticate, fun(_) -> {error, not_authorized} end), ok = meck:expect(emqx_flapping, detect, fun(_) -> Parent ! flapping_detect end), - IdleChannel = channel(#{conn_state => idle}), + IdleChannel = channel( + clientinfo(#{ + username => <<>>, + enable_authn => quick_deny_anonymous + }), + #{conn_state => idle} + ), {shutdown, not_authorized, _ConnAck, _Channel} = emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), IdleChannel), receive @@ -1000,7 +958,8 @@ t_flapping_detect(_) -> %%-------------------------------------------------------------------- channel() -> channel(#{}). -channel(InitFields) -> +channel(InitFields) -> channel(clientinfo(), InitFields). +channel(ClientInfo, InitFields) -> ConnInfo = #{ peername => {{127, 0, 0, 1}, 3456}, sockname => {{127, 0, 0, 1}, 1883}, @@ -1010,7 +969,7 @@ channel(InitFields) -> clean_start => true, keepalive => 30, clientid => <<"clientid">>, - username => <<"username">>, + username => maps:get(username, ClientInfo, <<"username">>), conn_props => #{}, receive_maximum => 100, expiry_interval => 0 @@ -1029,8 +988,8 @@ channel(InitFields) -> ), maps:merge( #{ - clientinfo => clientinfo(), - session => session(), + clientinfo => ClientInfo, + session => session(ClientInfo, #{}), conn_state => connected }, InitFields @@ -1045,6 +1004,7 @@ clientinfo(InitProps) -> listener => {tcp, default}, protocol => mqtt, peerhost => {127, 0, 0, 1}, + sockport => 3456, clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, @@ -1073,17 +1033,17 @@ connpkt(Props) -> session() -> session(#{zone => default, clientid => <<"fake-test">>}, #{}). session(InitFields) -> session(#{zone => default, clientid => <<"fake-test">>}, InitFields). session(ClientInfo, InitFields) when is_map(InitFields) -> - Conf = emqx_cm:get_session_confs( + Session = emqx_session:create( ClientInfo, #{ receive_maximum => 0, expiry_interval => 0 } ), - Session = emqx_session:init(Conf), maps:fold( fun(Field, Value, SessionAcc) -> - emqx_session:set_field(Field, Value, SessionAcc) + % TODO: assuming specific session implementation + emqx_session_mem:set_field(Field, Value, SessionAcc) end, Session, InitFields diff --git a/apps/emqx/test/emqx_cm_SUITE.erl b/apps/emqx/test/emqx_cm_SUITE.erl index 6cb58be46..6afdfa478 100644 --- a/apps/emqx/test/emqx_cm_SUITE.erl +++ b/apps/emqx/test/emqx_cm_SUITE.erl @@ -321,7 +321,7 @@ test_stepdown_session(Action, Reason) -> discard -> emqx_cm:discard_session(ClientId); {takeover, _} -> - none = emqx_cm:takeover_session(ClientId), + none = emqx_cm:takeover_session_begin(ClientId), ok end, case Reason =:= timeout orelse Reason =:= noproc of @@ -381,30 +381,35 @@ t_discard_session_race(_) -> t_takeover_session(_) -> #{conninfo := ConnInfo} = ?ChanInfo, - none = emqx_cm:takeover_session(<<"clientid">>), + ClientId = <<"clientid">>, + none = emqx_cm:takeover_session_begin(ClientId), Parent = self(), - erlang:spawn_link(fun() -> - ok = emqx_cm:register_channel(<<"clientid">>, self(), ConnInfo), + ChanPid = erlang:spawn_link(fun() -> + ok = emqx_cm:register_channel(ClientId, self(), ConnInfo), Parent ! registered, receive - {'$gen_call', From, {takeover, 'begin'}} -> - gen_server:reply(From, test), - ok + {'$gen_call', From1, {takeover, 'begin'}} -> + gen_server:reply(From1, test), + receive + {'$gen_call', From2, {takeover, 'end'}} -> + gen_server:reply(From2, _Pendings = []) + end end end), receive registered -> ok end, - {living, emqx_connection, _, test} = emqx_cm:takeover_session(<<"clientid">>), - emqx_cm:unregister_channel(<<"clientid">>). + {ok, test, State = {emqx_connection, ChanPid}} = emqx_cm:takeover_session_begin(ClientId), + {ok, []} = emqx_cm:takeover_session_end(State), + emqx_cm:unregister_channel(ClientId). t_takeover_session_process_gone(_) -> #{conninfo := ConnInfo} = ?ChanInfo, ClientIDTcp = <<"clientidTCP">>, ClientIDWs = <<"clientidWs">>, ClientIDRpc = <<"clientidRPC">>, - none = emqx_cm:takeover_session(ClientIDTcp), - none = emqx_cm:takeover_session(ClientIDWs), + none = emqx_cm:takeover_session_begin(ClientIDTcp), + none = emqx_cm:takeover_session_begin(ClientIDWs), meck:new(emqx_connection, [passthrough, no_history]), meck:expect( emqx_connection, @@ -417,7 +422,7 @@ t_takeover_session_process_gone(_) -> end ), ok = emqx_cm:register_channel(ClientIDTcp, self(), ConnInfo), - none = emqx_cm:takeover_session(ClientIDTcp), + none = emqx_cm:takeover_session_begin(ClientIDTcp), meck:expect( emqx_connection, call, @@ -429,7 +434,7 @@ t_takeover_session_process_gone(_) -> end ), ok = emqx_cm:register_channel(ClientIDWs, self(), ConnInfo), - none = emqx_cm:takeover_session(ClientIDWs), + none = emqx_cm:takeover_session_begin(ClientIDWs), meck:expect( emqx_connection, call, @@ -441,7 +446,7 @@ t_takeover_session_process_gone(_) -> end ), ok = emqx_cm:register_channel(ClientIDRpc, self(), ConnInfo), - none = emqx_cm:takeover_session(ClientIDRpc), + none = emqx_cm:takeover_session_begin(ClientIDRpc), emqx_cm:unregister_channel(ClientIDTcp), emqx_cm:unregister_channel(ClientIDWs), emqx_cm:unregister_channel(ClientIDRpc), diff --git a/apps/emqx/test/emqx_connection_SUITE.erl b/apps/emqx/test/emqx_connection_SUITE.erl index 01fe3c3db..83f5bce0f 100644 --- a/apps/emqx/test/emqx_connection_SUITE.erl +++ b/apps/emqx/test/emqx_connection_SUITE.erl @@ -676,10 +676,10 @@ channel(InitFields) -> is_superuser => false, mountpoint => undefined }, - Conf = emqx_cm:get_session_confs(ClientInfo, #{ - receive_maximum => 0, expiry_interval => 1000 - }), - Session = emqx_session:init(Conf), + Session = emqx_session:create( + ClientInfo, + #{receive_maximum => 0, expiry_interval => 1000} + ), maps:fold( fun(Field, Value, Channel) -> emqx_channel:set_field(Field, Value, Channel) diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index dddd096fa..24105b2b4 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -340,7 +340,6 @@ default_appspec(emqx_conf, SuiteOpts) -> node => #{ name => node(), cookie => erlang:get_cookie(), - % FIXME data_dir => unicode:characters_to_binary(maps:get(work_dir, SuiteOpts, "data")) } }, diff --git a/apps/emqx/test/emqx_inflight_SUITE.erl b/apps/emqx/test/emqx_inflight_SUITE.erl index 2c0949b88..a56e62575 100644 --- a/apps/emqx/test/emqx_inflight_SUITE.erl +++ b/apps/emqx/test/emqx_inflight_SUITE.erl @@ -76,6 +76,17 @@ t_values(_) -> ?assertEqual([1, 2], emqx_inflight:values(Inflight)), ?assertEqual([{a, 1}, {b, 2}], emqx_inflight:to_list(Inflight)). +t_fold(_) -> + Inflight = maps:fold( + fun emqx_inflight:insert/3, + emqx_inflight:new(), + #{a => 1, b => 2, c => 42} + ), + ?assertEqual( + emqx_inflight:fold(fun(_, V, S) -> S + V end, 0, Inflight), + lists:foldl(fun({_, V}, S) -> S + V end, 0, emqx_inflight:to_list(Inflight)) + ). + t_is_full(_) -> Inflight = emqx_inflight:insert(k, v, emqx_inflight:new()), ?assertNot(emqx_inflight:is_full(Inflight)), diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index 9d0f42424..751b7e4b8 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -186,13 +186,14 @@ t_session_subscription_iterators(Config) -> ct:pal("publishing 2"), Message2 = emqx_message:make(Topic, Payload2), publish(Node1, Message2), - [_] = receive_messages(1), + % TODO: no incoming publishes at the moment + % [_] = receive_messages(1), ct:pal("subscribing 2"), {ok, _, [1]} = emqtt:subscribe(Client, SubTopicFilter, qos1), ct:pal("publishing 3"), Message3 = emqx_message:make(Topic, Payload3), publish(Node1, Message3), - [_] = receive_messages(1), + % [_] = receive_messages(1), ct:pal("publishing 4"), Message4 = emqx_message:make(AnotherTopic, Payload4), publish(Node1, Message4), @@ -272,7 +273,7 @@ consume(Shard, IteratorId) when is_binary(IteratorId) -> consume(It) -> case emqx_ds_storage_layer:next(It) of {value, Msg, NIt} -> - [emqx_persistent_session_ds:deserialize_message(Msg) | consume(NIt)]; + [emqx_persistent_message:deserialize(Msg) | consume(NIt)]; none -> [] end. @@ -315,21 +316,13 @@ get_iterator_ids(Node, ClientId) -> app_specs() -> [ emqx_durable_storage, - {emqx, #{ - config => #{persistent_session_store => #{ds => true}}, - override_env => [{boot_modules, [broker, listeners]}] - }} + {emqx, "persistent_session_store {ds = true}"} ]. cluster() -> - Node1 = persistent_messages_SUITE1, - Spec = #{ - role => core, - join_to => emqx_cth_cluster:node_name(Node1), - apps => app_specs() - }, + Spec = #{role => core, apps => app_specs()}, [ - {Node1, Spec}, + {persistent_messages_SUITE1, Spec}, {persistent_messages_SUITE2, Spec} ]. diff --git a/apps/emqx/test/emqx_persistent_session_SUITE.erl b/apps/emqx/test/emqx_persistent_session_SUITE.erl index 07cfabc70..89fba9738 100644 --- a/apps/emqx/test/emqx_persistent_session_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_session_SUITE.erl @@ -19,8 +19,7 @@ -include_lib("stdlib/include/assert.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --include_lib("../include/emqx.hrl"). --include("../src/persistent_session/emqx_persistent_session.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -compile(export_all). -compile(nowarn_export_all). @@ -34,8 +33,8 @@ all() -> % NOTE % Tests are disabled while existing session persistence impl is being % phased out. - % {group, persistent_store_enabled}, - {group, persistent_store_disabled} + {group, persistent_store_disabled}, + {group, persistent_store_ds} ]. %% A persistent session can be resumed in two ways: @@ -51,177 +50,92 @@ all() -> groups() -> TCs = emqx_common_test_helpers:all(?MODULE), - SnabbkaffeTCs = [TC || TC <- TCs, is_snabbkaffe_tc(TC)], - GCTests = [TC || TC <- TCs, is_gc_tc(TC)], - OtherTCs = (TCs -- SnabbkaffeTCs) -- GCTests, + TCsNonGeneric = [t_choose_impl], [ - {persistent_store_enabled, [ - {group, ram_tables}, - {group, disc_tables} - ]}, {persistent_store_disabled, [{group, no_kill_connection_process}]}, - {ram_tables, [], [ - {group, no_kill_connection_process}, - {group, kill_connection_process}, - {group, snabbkaffe}, - {group, gc_tests} - ]}, - {disc_tables, [], [ - {group, no_kill_connection_process}, - {group, kill_connection_process}, - {group, snabbkaffe}, - {group, gc_tests} - ]}, + {persistent_store_ds, [{group, no_kill_connection_process}]}, {no_kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]}, - {kill_connection_process, [], [{group, tcp}, {group, quic}, {group, ws}]}, - {snabbkaffe, [], [ - {group, tcp_snabbkaffe}, {group, quic_snabbkaffe}, {group, ws_snabbkaffe} - ]}, - {tcp, [], OtherTCs}, - {quic, [], OtherTCs}, - {ws, [], OtherTCs}, - {tcp_snabbkaffe, [], SnabbkaffeTCs}, - {quic_snabbkaffe, [], SnabbkaffeTCs}, - {ws_snabbkaffe, [], SnabbkaffeTCs}, - {gc_tests, [], GCTests} + {tcp, [], TCs}, + {quic, [], TCs -- TCsNonGeneric}, + {ws, [], TCs -- TCsNonGeneric} ]. -is_snabbkaffe_tc(TC) -> - re:run(atom_to_list(TC), "^t_snabbkaffe_") /= nomatch. - -is_gc_tc(TC) -> - re:run(atom_to_list(TC), "^t_gc_") /= nomatch. - -init_per_group(persistent_store_enabled, Config) -> - [{persistent_store_enabled, true} | Config]; -init_per_group(Group, Config) when Group =:= ram_tables; Group =:= disc_tables -> - %% Start Apps - Reply = - case Group =:= ram_tables of - true -> ram; - false -> disc - end, - emqx_common_test_helpers:boot_modules(all), - meck:new(emqx_config, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_config, get, fun - (?on_disc_key) -> Reply =:= disc; - (?is_enabled_key) -> true; - (Other) -> meck:passthrough([Other]) - end), - emqx_common_test_helpers:start_apps([], fun set_special_confs/1), - ?assertEqual(true, emqx_persistent_session:is_store_enabled()), - Config; init_per_group(persistent_store_disabled, Config) -> - %% Start Apps - emqx_common_test_helpers:boot_modules(all), - meck:new(emqx_config, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_config, get, fun - (?is_enabled_key) -> false; - (Other) -> meck:passthrough([Other]) - end), - emqx_common_test_helpers:start_apps([], fun set_special_confs/1), - ?assertEqual(false, emqx_persistent_session:is_store_enabled()), - [{persistent_store_enabled, false} | Config]; -init_per_group(Group, Config) when Group == ws; Group == ws_snabbkaffe -> + [ + {emqx_config, "persistent_session_store { enabled = false }"}, + {persistent_store, false} + | Config + ]; +init_per_group(persistent_store_ds, Config) -> + [ + {emqx_config, "persistent_session_store { ds = true }"}, + {persistent_store, ds} + | Config + ]; +init_per_group(Group, Config) when Group == tcp -> + Apps = emqx_cth_suite:start( + [{emqx, ?config(emqx_config, Config)}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [ + {port, get_listener_port(tcp, default)}, + {conn_fun, connect}, + {group_apps, Apps} + | Config + ]; +init_per_group(Group, Config) when Group == ws -> + Apps = emqx_cth_suite:start( + [{emqx, ?config(emqx_config, Config)}], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), [ {ssl, false}, {host, "localhost"}, {enable_websocket, true}, - {port, 8083}, - {conn_fun, ws_connect} + {port, get_listener_port(ws, default)}, + {conn_fun, ws_connect}, + {group_apps, Apps} + | Config + ]; +init_per_group(Group, Config) when Group == quic -> + Apps = emqx_cth_suite:start( + [ + {emqx, + ?config(emqx_config, Config) ++ + "\n listeners.quic.test { enable = true }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [ + {port, get_listener_port(quic, test)}, + {conn_fun, quic_connect}, + {group_apps, Apps} | Config ]; -init_per_group(Group, Config) when Group == tcp; Group == tcp_snabbkaffe -> - [{port, 1883}, {conn_fun, connect} | Config]; -init_per_group(Group, Config) when Group == quic; Group == quic_snabbkaffe -> - UdpPort = 1883, - emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort), - [{port, UdpPort}, {conn_fun, quic_connect} | Config]; init_per_group(no_kill_connection_process, Config) -> [{kill_connection_process, false} | Config]; init_per_group(kill_connection_process, Config) -> - [{kill_connection_process, true} | Config]; -init_per_group(snabbkaffe, Config) -> - [{kill_connection_process, true} | Config]; -init_per_group(gc_tests, Config) -> - %% We need to make sure the system does not interfere with this test group. - lists:foreach( - fun(ClientId) -> - maybe_kill_connection_process(ClientId, [{kill_connection_process, true}]) - end, - emqx_cm:all_client_ids() - ), - emqx_common_test_helpers:stop_apps([]), - SessionMsgEts = gc_tests_session_store, - MsgEts = gc_tests_msg_store, - Pid = spawn(fun() -> - ets:new(SessionMsgEts, [named_table, public, ordered_set]), - ets:new(MsgEts, [named_table, public, ordered_set, {keypos, 2}]), - receive - stop -> ok - end - end), - meck:new(mnesia, [non_strict, passthrough, no_history, no_link]), - meck:expect(mnesia, dirty_first, fun - (?SESS_MSG_TAB) -> ets:first(SessionMsgEts); - (?MSG_TAB) -> ets:first(MsgEts); - (X) -> meck:passthrough([X]) - end), - meck:expect(mnesia, dirty_next, fun - (?SESS_MSG_TAB, X) -> ets:next(SessionMsgEts, X); - (?MSG_TAB, X) -> ets:next(MsgEts, X); - (Tab, X) -> meck:passthrough([Tab, X]) - end), - meck:expect(mnesia, dirty_delete, fun - (?MSG_TAB, X) -> ets:delete(MsgEts, X); - (Tab, X) -> meck:passthrough([Tab, X]) - end), - [{store_owner, Pid}, {session_msg_store, SessionMsgEts}, {msg_store, MsgEts} | Config]. + [{kill_connection_process, true} | Config]. -init_per_suite(Config) -> - Config. +get_listener_port(Type, Name) -> + case emqx_config:get([listeners, Type, Name, bind]) of + {_, Port} -> Port; + Port -> Port + end. -set_special_confs(_) -> - ok. - -end_per_suite(_Config) -> - emqx_common_test_helpers:ensure_mnesia_stopped(), - ok. - -end_per_group(gc_tests, Config) -> - meck:unload(mnesia), - ?config(store_owner, Config) ! stop, - ok; -end_per_group(Group, _Config) when - Group =:= ram_tables; Group =:= disc_tables --> - meck:unload(emqx_config), - emqx_common_test_helpers:stop_apps([]); -end_per_group(persistent_store_disabled, _Config) -> - meck:unload(emqx_config), - emqx_common_test_helpers:stop_apps([]); -end_per_group(_Group, _Config) -> +end_per_group(Group, Config) when Group == tcp; Group == ws; Group == quic -> + ok = emqx_cth_suite:stop(?config(group_apps, Config)); +end_per_group(_, _Config) -> ok. init_per_testcase(TestCase, Config) -> Config1 = preconfig_per_testcase(TestCase, Config), - case is_gc_tc(TestCase) of - true -> - ets:delete_all_objects(?config(msg_store, Config)), - ets:delete_all_objects(?config(session_msg_store, Config)); - false -> - skip - end, case erlang:function_exported(?MODULE, TestCase, 2) of true -> ?MODULE:TestCase(init, Config1); _ -> Config1 end. end_per_testcase(TestCase, Config) -> - case is_snabbkaffe_tc(TestCase) of - true -> snabbkaffe:stop(); - false -> skip - end, case erlang:function_exported(?MODULE, TestCase, 2) of true -> ?MODULE:TestCase('end', Config); false -> ok @@ -307,20 +221,6 @@ wait_for_cm_unregister(ClientId, N) -> wait_for_cm_unregister(ClientId, N - 1) end. -snabbkaffe_sync_publish(Topic, Payloads) -> - Fun = fun(Client, Payload) -> - ?check_trace( - begin - ?wait_async_action( - {ok, _} = emqtt:publish(Client, Topic, Payload, 2), - #{?snk_kind := ps_persist_msg, payload := Payload} - ) - end, - fun(_, _Trace) -> ok end - ) - end, - do_publish(Payloads, Fun, true). - publish(Topic, Payloads) -> publish(Topic, Payloads, false). @@ -377,7 +277,55 @@ do_publish(Payload, PublishFun, WaitForUnregister) -> %% Test Cases %%-------------------------------------------------------------------- +t_choose_impl(Config) -> + ClientId = ?config(client_id, Config), + ConnFun = ?config(conn_fun, Config), + {ok, Client} = emqtt:start_link([ + {clientid, ClientId}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 30}} + | Config + ]), + {ok, _} = emqtt:ConnFun(Client), + [ChanPid] = emqx_cm:lookup_channels(ClientId), + ?assertEqual( + case ?config(persistent_store, Config) of + false -> emqx_session_mem; + ds -> emqx_persistent_session_ds + end, + emqx_connection:info({channel, {session, impl}}, sys:get_state(ChanPid)) + ). + +t_connect_discards_existing_client(Config) -> + ClientId = ?config(client_id, Config), + ConnFun = ?config(conn_fun, Config), + ClientOpts = [ + {clientid, ClientId}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 30}} + | Config + ], + + {ok, Client1} = emqtt:start_link(ClientOpts), + true = unlink(Client1), + MRef = erlang:monitor(process, Client1), + {ok, _} = emqtt:ConnFun(Client1), + + {ok, Client2} = emqtt:start_link(ClientOpts), + {ok, _} = emqtt:ConnFun(Client2), + + receive + {'DOWN', MRef, process, Client1, Reason} -> + ok = ?assertMatch({disconnected, ?RC_SESSION_TAKEN_OVER, _}, Reason), + ok = emqtt:stop(Client2), + ok + after 1000 -> + error({client_still_connected, Client1}) + end. + %% [MQTT-3.1.2-23] +t_connect_session_expiry_interval(init, Config) -> skip_ds_tc(Config); +t_connect_session_expiry_interval('end', _Config) -> ok. t_connect_session_expiry_interval(Config) -> ConnFun = ?config(conn_fun, Config), Topic = ?config(topic, Config), @@ -514,20 +462,8 @@ t_persist_on_disconnect(Config) -> ?assertEqual(0, client_info(session_present, Client2)), ok = emqtt:disconnect(Client2). -wait_for_pending(SId) -> - wait_for_pending(SId, 100). - -wait_for_pending(_SId, 0) -> - error(exhausted_wait_for_pending); -wait_for_pending(SId, N) -> - case emqx_persistent_session:pending(SId) of - [] -> - timer:sleep(1), - wait_for_pending(SId, N - 1); - [_ | _] = Pending -> - Pending - end. - +t_process_dies_session_expires(init, Config) -> skip_ds_tc(Config); +t_process_dies_session_expires('end', _Config) -> ok. t_process_dies_session_expires(Config) -> %% Emulate an error in the connect process, %% or that the node of the process goes down. @@ -552,36 +488,8 @@ t_process_dies_session_expires(Config) -> ok = publish(Topic, [Payload]), - SessionId = - case ?config(persistent_store_enabled, Config) of - false -> - undefined; - true -> - %% The session should not be marked as expired. - {Tag, Session} = emqx_persistent_session:lookup(ClientId), - ?assertEqual(persistent, Tag), - SId = emqx_session:info(id, Session), - case ?config(kill_connection_process, Config) of - true -> - %% The session should have a pending message - ?assertMatch([_], wait_for_pending(SId)); - false -> - skip - end, - SId - end, - timer:sleep(1100), - %% The session should now be marked as expired. - case - (?config(kill_connection_process, Config) andalso - ?config(persistent_store_enabled, Config)) - of - true -> ?assertMatch({expired, _}, emqx_persistent_session:lookup(ClientId)); - false -> skip - end, - {ok, Client2} = emqtt:start_link([ {proto_ver, v5}, {clientid, ClientId}, @@ -592,26 +500,13 @@ t_process_dies_session_expires(Config) -> {ok, _} = emqtt:ConnFun(Client2), ?assertEqual(0, client_info(session_present, Client2)), - case - (?config(kill_connection_process, Config) andalso - ?config(persistent_store_enabled, Config)) - of - true -> - %% The session should be a fresh one - {persistent, NewSession} = emqx_persistent_session:lookup(ClientId), - ?assertNotEqual(SessionId, emqx_session:info(id, NewSession)), - %% The old session should now either - %% be marked as abandoned or already be garbage collected. - ?assertMatch([], emqx_persistent_session:pending(SessionId)); - false -> - skip - end, - %% We should not receive the pending message ?assertEqual([], receive_messages(1)), emqtt:disconnect(Client2). +t_publish_while_client_is_gone(init, Config) -> skip_ds_tc(Config); +t_publish_while_client_is_gone('end', _Config) -> ok. t_publish_while_client_is_gone(Config) -> %% A persistent session should receive messages in its %% subscription even if the process owning the session dies. @@ -654,6 +549,8 @@ t_publish_while_client_is_gone(Config) -> ok = emqtt:disconnect(Client2). +t_clean_start_drops_subscriptions(init, Config) -> skip_ds_tc(Config); +t_clean_start_drops_subscriptions('end', _Config) -> ok. t_clean_start_drops_subscriptions(Config) -> %% 1. A persistent session is started and disconnected. %% 2. While disconnected, a message is published and persisted. @@ -724,7 +621,6 @@ t_clean_start_drops_subscriptions(Config) -> t_unsubscribe(Config) -> ConnFun = ?config(conn_fun, Config), - Topic = ?config(topic, Config), STopic = ?config(stopic, Config), ClientId = ?config(client_id, Config), {ok, Client} = emqtt:start_link([ @@ -735,24 +631,13 @@ t_unsubscribe(Config) -> ]), {ok, _} = emqtt:ConnFun(Client), {ok, _, [2]} = emqtt:subscribe(Client, STopic, qos2), - case emqx_persistent_session:is_store_enabled() of - true -> - {persistent, Session} = emqx_persistent_session:lookup(ClientId), - SessionID = emqx_session:info(id, Session), - SessionIDs = [SId || #route{dest = SId} <- emqx_session_router:match_routes(Topic)], - ?assert(lists:member(SessionID, SessionIDs)), - ?assertMatch([_], [Sub || {ST, _} = Sub <- emqtt:subscriptions(Client), ST =:= STopic]), - {ok, _, _} = emqtt:unsubscribe(Client, STopic), - ?assertMatch([], [Sub || {ST, _} = Sub <- emqtt:subscriptions(Client), ST =:= STopic]), - SessionIDs2 = [SId || #route{dest = SId} <- emqx_session_router:match_routes(Topic)], - ?assert(not lists:member(SessionID, SessionIDs2)); - false -> - ?assertMatch([_], [Sub || {ST, _} = Sub <- emqtt:subscriptions(Client), ST =:= STopic]), - {ok, _, _} = emqtt:unsubscribe(Client, STopic), - ?assertMatch([], [Sub || {ST, _} = Sub <- emqtt:subscriptions(Client), ST =:= STopic]) - end, + ?assertMatch([_], [Sub || {ST, _} = Sub <- emqtt:subscriptions(Client), ST =:= STopic]), + {ok, _, _} = emqtt:unsubscribe(Client, STopic), + ?assertMatch([], [Sub || {ST, _} = Sub <- emqtt:subscriptions(Client), ST =:= STopic]), ok = emqtt:disconnect(Client). +t_multiple_subscription_matches(init, Config) -> skip_ds_tc(Config); +t_multiple_subscription_matches('end', _Config) -> ok. t_multiple_subscription_matches(Config) -> ConnFun = ?config(conn_fun, Config), Topic = ?config(topic, Config), @@ -795,514 +680,10 @@ t_multiple_subscription_matches(Config) -> ?assertEqual({ok, 2}, maps:find(qos, Msg2)), ok = emqtt:disconnect(Client2). -t_lost_messages_because_of_gc(init, Config) -> - case - (emqx_persistent_session:is_store_enabled() andalso - ?config(kill_connection_process, Config)) - of - true -> - Retain = 1000, - OldRetain = emqx_config:get(?msg_retain, Retain), - emqx_config:put(?msg_retain, Retain), - [{retain, Retain}, {old_retain, OldRetain} | Config]; - false -> - {skip, only_relevant_with_store_and_kill_process} - end; -t_lost_messages_because_of_gc('end', Config) -> - OldRetain = ?config(old_retain, Config), - emqx_config:put(?msg_retain, OldRetain), - ok. - -t_lost_messages_because_of_gc(Config) -> - ConnFun = ?config(conn_fun, Config), - Topic = ?config(topic, Config), - STopic = ?config(stopic, Config), - ClientId = ?config(client_id, Config), - Retain = ?config(retain, Config), - Payload1 = <<"hello1">>, - Payload2 = <<"hello2">>, - {ok, Client1} = emqtt:start_link([ - {clientid, ClientId}, - {proto_ver, v5}, - {properties, #{'Session-Expiry-Interval' => 30}} - | Config - ]), - {ok, _} = emqtt:ConnFun(Client1), - {ok, _, [2]} = emqtt:subscribe(Client1, STopic, qos2), - emqtt:disconnect(Client1), - maybe_kill_connection_process(ClientId, Config), - publish(Topic, Payload1), - timer:sleep(2 * Retain), - publish(Topic, Payload2), - emqx_persistent_session_gc:message_gc_worker(), - {ok, Client2} = emqtt:start_link([ - {clientid, ClientId}, - {clean_start, false}, - {proto_ver, v5}, - {properties, #{'Session-Expiry-Interval' => 30}} - | Config - ]), - {ok, _} = emqtt:ConnFun(Client2), - Msgs = receive_messages(2), - ?assertMatch([_], Msgs), - ?assertEqual({ok, iolist_to_binary(Payload2)}, maps:find(payload, hd(Msgs))), - emqtt:disconnect(Client2), - ok. - -%%-------------------------------------------------------------------- -%% Snabbkaffe helpers -%%-------------------------------------------------------------------- - -check_snabbkaffe_vanilla(Trace) -> - ResumeTrace = [ - T - || #{?snk_kind := K} = T <- Trace, - re:run(to_list(K), "^ps_") /= nomatch - ], - ?assertMatch([_ | _], ResumeTrace), - [_Sid] = lists:usort(?projection(sid, ResumeTrace)), - %% Check internal flow of the emqx_cm resuming - ?assert( - ?strict_causality( - #{?snk_kind := ps_resuming}, - #{?snk_kind := ps_initial_pendings}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_initial_pendings}, - #{?snk_kind := ps_persist_pendings}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_persist_pendings}, - #{?snk_kind := ps_notify_writers}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_notify_writers}, - #{?snk_kind := ps_node_markers}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_node_markers}, - #{?snk_kind := ps_resume_session}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_resume_session}, - #{?snk_kind := ps_marker_pendings}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_marker_pendings}, - #{?snk_kind := ps_marker_pendings_msgs}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_marker_pendings_msgs}, - #{?snk_kind := ps_resume_end}, - ResumeTrace - ) - ), - - %% Check flow between worker and emqx_cm - ?assert( - ?strict_causality( - #{?snk_kind := ps_notify_writers}, - #{?snk_kind := ps_worker_started}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_marker_pendings}, - #{?snk_kind := ps_worker_resume_end}, - ResumeTrace - ) - ), - ?assert( - ?strict_causality( - #{?snk_kind := ps_worker_resume_end}, - #{?snk_kind := ps_worker_shutdown}, - ResumeTrace - ) - ), - - [Markers] = ?projection(markers, ?of_kind(ps_node_markers, Trace)), - ?assertMatch([_], Markers). - -to_list(L) when is_list(L) -> L; -to_list(A) when is_atom(A) -> atom_to_list(A); -to_list(B) when is_binary(B) -> binary_to_list(B). - -%%-------------------------------------------------------------------- -%% Snabbkaffe tests -%%-------------------------------------------------------------------- - -t_snabbkaffe_vanilla_stages(Config) -> - %% Test that all stages of session resume works ok in the simplest case - ConnFun = ?config(conn_fun, Config), - ClientId = ?config(client_id, Config), - EmqttOpts = [ - {proto_ver, v5}, - {clientid, ClientId}, - {properties, #{'Session-Expiry-Interval' => 30}} - | Config - ], - {ok, Client1} = emqtt:start_link([{clean_start, true} | EmqttOpts]), - {ok, _} = emqtt:ConnFun(Client1), - ok = emqtt:disconnect(Client1), - maybe_kill_connection_process(ClientId, Config), - - ?check_trace( - begin - {ok, Client2} = emqtt:start_link([{clean_start, false} | EmqttOpts]), - {ok, _} = emqtt:ConnFun(Client2), - ok = emqtt:disconnect(Client2) - end, - fun(ok, Trace) -> - check_snabbkaffe_vanilla(Trace) - end - ), - ok. - -t_snabbkaffe_pending_messages(Config) -> - %% Make sure there are pending messages are fetched during the init stage. - ConnFun = ?config(conn_fun, Config), - ClientId = ?config(client_id, Config), - Topic = ?config(topic, Config), - STopic = ?config(stopic, Config), - Payloads = [<<"test", (integer_to_binary(X))/binary>> || X <- [1, 2, 3, 4, 5]], - EmqttOpts = [ - {proto_ver, v5}, - {clientid, ClientId}, - {properties, #{'Session-Expiry-Interval' => 30}} - | Config - ], - {ok, Client1} = emqtt:start_link([{clean_start, true} | EmqttOpts]), - {ok, _} = emqtt:ConnFun(Client1), - {ok, _, [2]} = emqtt:subscribe(Client1, STopic, qos2), - ok = emqtt:disconnect(Client1), - maybe_kill_connection_process(ClientId, Config), - - ?check_trace( - begin - snabbkaffe_sync_publish(Topic, Payloads), - {ok, Client2} = emqtt:start_link([{clean_start, false} | EmqttOpts]), - {ok, _} = emqtt:ConnFun(Client2), - Msgs = receive_messages(length(Payloads)), - ReceivedPayloads = [P || #{payload := P} <- Msgs], - ?assertEqual(lists:sort(ReceivedPayloads), lists:sort(Payloads)), - ok = emqtt:disconnect(Client2) - end, - fun(ok, Trace) -> - check_snabbkaffe_vanilla(Trace), - %% Check that all messages was delivered from the DB - [Delivers1] = ?projection(msgs, ?of_kind(ps_persist_pendings_msgs, Trace)), - [Delivers2] = ?projection(msgs, ?of_kind(ps_marker_pendings_msgs, Trace)), - Delivers = Delivers1 ++ Delivers2, - ?assertEqual(length(Payloads), length(Delivers)), - %% Check for no duplicates - ?assertEqual(lists:usort(Delivers), lists:sort(Delivers)) - end - ), - ok. - -t_snabbkaffe_buffered_messages(Config) -> - %% Make sure to buffer messages during startup. - ConnFun = ?config(conn_fun, Config), - ClientId = ?config(client_id, Config), - Topic = ?config(topic, Config), - STopic = ?config(stopic, Config), - Payloads1 = [<<"test", (integer_to_binary(X))/binary>> || X <- [1, 2, 3]], - Payloads2 = [<<"test", (integer_to_binary(X))/binary>> || X <- [4, 5, 6]], - EmqttOpts = [ - {proto_ver, v5}, - {clientid, ClientId}, - {properties, #{'Session-Expiry-Interval' => 30}} - | Config - ], - {ok, Client1} = emqtt:start_link([{clean_start, true} | EmqttOpts]), - {ok, _} = emqtt:ConnFun(Client1), - {ok, _, [2]} = emqtt:subscribe(Client1, STopic, qos2), - ok = emqtt:disconnect(Client1), - maybe_kill_connection_process(ClientId, Config), - - publish(Topic, Payloads1), - - ?check_trace( - begin - %% Make the resume init phase wait until the first message is delivered. - ?force_ordering( - #{?snk_kind := ps_worker_deliver}, - #{?snk_kind := ps_resume_end} - ), - Parent = self(), - spawn_link(fun() -> - ?block_until(#{?snk_kind := ps_marker_pendings_msgs}, infinity, 5000), - publish(Topic, Payloads2, true), - Parent ! publish_done, - ok - end), - {ok, Client2} = emqtt:start_link([{clean_start, false} | EmqttOpts]), - {ok, _} = emqtt:ConnFun(Client2), - receive - publish_done -> ok - after 10000 -> error(too_long_to_publish) - end, - Msgs = receive_messages(length(Payloads1) + length(Payloads2) + 1), - ReceivedPayloads = [P || #{payload := P} <- Msgs], - ?assertEqual( - lists:sort(Payloads1 ++ Payloads2), - lists:sort(ReceivedPayloads) - ), - ok = emqtt:disconnect(Client2) - end, - fun(ok, Trace) -> - check_snabbkaffe_vanilla(Trace), - %% Check that some messages was buffered in the writer process - [Msgs] = ?projection(msgs, ?of_kind(ps_writer_pendings, Trace)), - ?assertMatch( - X when 0 < X andalso X =< length(Payloads2), - length(Msgs) - ) - end - ), - ok. - -%%-------------------------------------------------------------------- -%% GC tests -%%-------------------------------------------------------------------- - --define(MARKER, 3). --define(DELIVERED, 2). --define(UNDELIVERED, 1). --define(ABANDONED, 0). - -msg_id() -> - emqx_guid:gen(). - -delivered_msg(MsgId, SessionID, STopic) -> - {SessionID, MsgId, STopic, ?DELIVERED}. - -undelivered_msg(MsgId, SessionID, STopic) -> - {SessionID, MsgId, STopic, ?UNDELIVERED}. - -marker_msg(MarkerID, SessionID) -> - {SessionID, MarkerID, <<>>, ?MARKER}. - -guid(MicrosecondsAgo) -> - %% Make a fake GUID and set a timestamp. - <> = emqx_guid:gen(), - <<(TS - MicrosecondsAgo):64, Tail:64>>. - -abandoned_session_msg(SessionID) -> - abandoned_session_msg(SessionID, 0). - -abandoned_session_msg(SessionID, MicrosecondsAgo) -> - TS = erlang:system_time(microsecond), - {SessionID, <<>>, <<(TS - MicrosecondsAgo):64>>, ?ABANDONED}. - -fresh_gc_delete_fun() -> - Ets = ets:new(gc_collect, [ordered_set]), - fun - (delete, Key) -> - ets:insert(Ets, {Key}), - ok; - (collect, <<>>) -> - List = ets:match(Ets, {'$1'}), - ets:delete(Ets), - lists:append(List); - (_, _Key) -> - ok +skip_ds_tc(Config) -> + case ?config(persistent_store, Config) of + ds -> + {skip, "Testcase not yet supported under 'emqx_persistent_session_ds' implementation"}; + _ -> + Config end. - -fresh_gc_callbacks_fun() -> - Ets = ets:new(gc_collect, [ordered_set]), - fun - (collect, <<>>) -> - List = ets:match(Ets, {'$1'}), - ets:delete(Ets), - lists:append(List); - (Tag, Key) -> - ets:insert(Ets, {{Key, Tag}}), - ok - end. - -get_gc_delete_messages() -> - Fun = fresh_gc_delete_fun(), - emqx_persistent_session:gc_session_messages(Fun), - Fun(collect, <<>>). - -get_gc_callbacks() -> - Fun = fresh_gc_callbacks_fun(), - emqx_persistent_session:gc_session_messages(Fun), - Fun(collect, <<>>). - -t_gc_all_delivered(Config) -> - Store = ?config(session_msg_store, Config), - STopic = ?config(stopic, Config), - SessionId = emqx_guid:gen(), - MsgIds = [msg_id() || _ <- lists:seq(1, 5)], - Delivered = [delivered_msg(X, SessionId, STopic) || X <- MsgIds], - Undelivered = [undelivered_msg(X, SessionId, STopic) || X <- MsgIds], - SortedContent = lists:usort(Delivered ++ Undelivered), - ets:insert(Store, [{X, <<>>} || X <- SortedContent]), - GCMessages = get_gc_delete_messages(), - ?assertEqual(SortedContent, GCMessages), - ok. - -t_gc_some_undelivered(Config) -> - Store = ?config(session_msg_store, Config), - STopic = ?config(stopic, Config), - SessionId = emqx_guid:gen(), - MsgIds = [msg_id() || _ <- lists:seq(1, 10)], - Delivered = [delivered_msg(X, SessionId, STopic) || X <- MsgIds], - {Delivered1, _Delivered2} = split(Delivered), - Undelivered = [undelivered_msg(X, SessionId, STopic) || X <- MsgIds], - {Undelivered1, Undelivered2} = split(Undelivered), - Content = Delivered1 ++ Undelivered1 ++ Undelivered2, - ets:insert(Store, [{X, <<>>} || X <- Content]), - Expected = lists:usort(Delivered1 ++ Undelivered1), - GCMessages = get_gc_delete_messages(), - ?assertEqual(Expected, GCMessages), - ok. - -t_gc_with_markers(Config) -> - Store = ?config(session_msg_store, Config), - STopic = ?config(stopic, Config), - SessionId = emqx_guid:gen(), - MsgIds1 = [msg_id() || _ <- lists:seq(1, 10)], - MarkerId = msg_id(), - MsgIds = [msg_id() || _ <- lists:seq(1, 4)] ++ MsgIds1, - Delivered = [delivered_msg(X, SessionId, STopic) || X <- MsgIds], - {Delivered1, _Delivered2} = split(Delivered), - Undelivered = [undelivered_msg(X, SessionId, STopic) || X <- MsgIds], - {Undelivered1, Undelivered2} = split(Undelivered), - Markers = [marker_msg(MarkerId, SessionId)], - Content = Delivered1 ++ Undelivered1 ++ Undelivered2 ++ Markers, - ets:insert(Store, [{X, <<>>} || X <- Content]), - Expected = lists:usort(Delivered1 ++ Undelivered1), - GCMessages = get_gc_delete_messages(), - ?assertEqual(Expected, GCMessages), - ok. - -t_gc_abandoned_some_undelivered(Config) -> - Store = ?config(session_msg_store, Config), - STopic = ?config(stopic, Config), - SessionId = emqx_guid:gen(), - MsgIds = [msg_id() || _ <- lists:seq(1, 10)], - Delivered = [delivered_msg(X, SessionId, STopic) || X <- MsgIds], - {Delivered1, _Delivered2} = split(Delivered), - Undelivered = [undelivered_msg(X, SessionId, STopic) || X <- MsgIds], - {Undelivered1, Undelivered2} = split(Undelivered), - Abandoned = abandoned_session_msg(SessionId), - Content = Delivered1 ++ Undelivered1 ++ Undelivered2 ++ [Abandoned], - ets:insert(Store, [{X, <<>>} || X <- Content]), - Expected = lists:usort(Delivered1 ++ Undelivered1 ++ Undelivered2), - GCMessages = get_gc_delete_messages(), - ?assertEqual(Expected, GCMessages), - ok. - -t_gc_abandoned_only_called_on_empty_session(Config) -> - Store = ?config(session_msg_store, Config), - STopic = ?config(stopic, Config), - SessionId = emqx_guid:gen(), - MsgIds = [msg_id() || _ <- lists:seq(1, 10)], - Delivered = [delivered_msg(X, SessionId, STopic) || X <- MsgIds], - Undelivered = [undelivered_msg(X, SessionId, STopic) || X <- MsgIds], - Abandoned = abandoned_session_msg(SessionId), - Content = Delivered ++ Undelivered ++ [Abandoned], - ets:insert(Store, [{X, <<>>} || X <- Content]), - GCMessages = get_gc_callbacks(), - - %% Since we had messages to delete, we don't expect to get the - %% callback on the abandoned session - ?assertEqual([], [X || {X, abandoned} <- GCMessages]), - - %% But if we have only the abandoned session marker for this - %% session, it should be called. - ets:delete_all_objects(Store), - UndeliveredOtherSession = undelivered_msg(msg_id(), emqx_guid:gen(), <<"topic">>), - ets:insert(Store, [{X, <<>>} || X <- [Abandoned, UndeliveredOtherSession]]), - GCMessages2 = get_gc_callbacks(), - ?assertEqual([Abandoned], [X || {X, abandoned} <- GCMessages2]), - ok. - -t_gc_session_gc_worker(init, Config) -> - meck:new(emqx_persistent_session, [passthrough, no_link]), - Config; -t_gc_session_gc_worker('end', _Config) -> - meck:unload(emqx_persistent_session), - ok. - -t_gc_session_gc_worker(Config) -> - STopic = ?config(stopic, Config), - SessionID = emqx_guid:gen(), - MsgDeleted = delivered_msg(msg_id(), SessionID, STopic), - MarkerNotDeleted = marker_msg(msg_id(), SessionID), - MarkerDeleted = marker_msg(guid(120 * 1000 * 1000), SessionID), - AbandonedNotDeleted = abandoned_session_msg(SessionID), - AbandonedDeleted = abandoned_session_msg(SessionID, 500 * 1000 * 1000), - meck:expect(emqx_persistent_session, delete_session_message, fun(_Key) -> ok end), - emqx_persistent_session_gc:session_gc_worker(delete, MsgDeleted), - emqx_persistent_session_gc:session_gc_worker(marker, MarkerNotDeleted), - emqx_persistent_session_gc:session_gc_worker(marker, MarkerDeleted), - emqx_persistent_session_gc:session_gc_worker(abandoned, AbandonedDeleted), - emqx_persistent_session_gc:session_gc_worker(abandoned, AbandonedNotDeleted), - History = meck:history(emqx_persistent_session, self()), - DeleteCalls = [ - Key - || {_Pid, {_, delete_session_message, [Key]}, _Result} <- - History - ], - ?assertEqual( - lists:sort([MsgDeleted, AbandonedDeleted, MarkerDeleted]), - lists:sort(DeleteCalls) - ), - ok. - -t_gc_message_gc(Config) -> - Topic = ?config(topic, Config), - ClientID = ?config(client_id, Config), - Store = ?config(msg_store, Config), - NewMsgs = [ - emqx_message:make(ClientID, Topic, integer_to_binary(P)) - || P <- lists:seq(6, 10) - ], - Retain = 60 * 1000, - emqx_config:put(?msg_retain, Retain), - Msgs1 = [ - emqx_message:make(ClientID, Topic, integer_to_binary(P)) - || P <- lists:seq(1, 5) - ], - OldMsgs = [M#message{id = guid(Retain * 1000)} || M <- Msgs1], - ets:insert(Store, NewMsgs ++ OldMsgs), - ?assertEqual(lists:sort(OldMsgs ++ NewMsgs), ets:tab2list(Store)), - ok = emqx_persistent_session_gc:message_gc_worker(), - ?assertEqual(lists:sort(NewMsgs), ets:tab2list(Store)), - ok. - -split(List) -> - split(List, [], []). - -split([], L1, L2) -> - {L1, L2}; -split([H], L1, L2) -> - {[H | L1], L2}; -split([H1, H2 | Left], L1, L2) -> - split(Left, [H1 | L1], [H2 | L2]). diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index ab1720754..0a66b3628 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -20,7 +20,7 @@ -include_lib("proper/include/proper.hrl"). -include("emqx.hrl"). --include("emqx_session.hrl"). +-include("emqx_session_mem.hrl"). -include("emqx_access_control.hrl"). %% High level Types diff --git a/apps/emqx/test/emqx_session_SUITE.erl b/apps/emqx/test/emqx_session_SUITE.erl deleted file mode 100644 index 88fae7156..000000000 --- a/apps/emqx/test/emqx_session_SUITE.erl +++ /dev/null @@ -1,527 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_session_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("eunit/include/eunit.hrl"). - -all() -> emqx_common_test_helpers:all(?MODULE). - --define(NOW, erlang:system_time(millisecond)). - --type inflight_data_phase() :: wait_ack | wait_comp. - --record(inflight_data, { - phase :: inflight_data_phase(), - message :: emqx_types:message(), - timestamp :: non_neg_integer() -}). - -%%-------------------------------------------------------------------- -%% CT callbacks -%%-------------------------------------------------------------------- - -init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([]), - ok = meck:new( - [emqx_hooks, emqx_metrics, emqx_broker], - [passthrough, no_history, no_link] - ), - ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end), - ok = meck:expect(emqx_metrics, inc, fun(_K, _V) -> ok end), - ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end), - Config. - -end_per_suite(_Config) -> - meck:unload([emqx_broker, emqx_hooks, emqx_metrics]). - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, Config) -> - Config. - -%%-------------------------------------------------------------------- -%% Test cases for session init -%%-------------------------------------------------------------------- - -t_session_init(_) -> - Conf = emqx_cm:get_session_confs( - #{zone => default, clientid => <<"fake-test">>}, #{ - receive_maximum => 64, expiry_interval => 0 - } - ), - Session = emqx_session:init(Conf), - ?assertEqual(#{}, emqx_session:info(subscriptions, Session)), - ?assertEqual(0, emqx_session:info(subscriptions_cnt, Session)), - ?assertEqual(infinity, emqx_session:info(subscriptions_max, Session)), - ?assertEqual(false, emqx_session:info(upgrade_qos, Session)), - ?assertEqual(0, emqx_session:info(inflight_cnt, Session)), - ?assertEqual(64, emqx_session:info(inflight_max, Session)), - ?assertEqual(1, emqx_session:info(next_pkt_id, Session)), - ?assertEqual(30000, emqx_session:info(retry_interval, Session)), - ?assertEqual(0, emqx_mqueue:len(emqx_session:info(mqueue, Session))), - ?assertEqual(0, emqx_session:info(awaiting_rel_cnt, Session)), - ?assertEqual(100, emqx_session:info(awaiting_rel_max, Session)), - ?assertEqual(300000, emqx_session:info(await_rel_timeout, Session)), - ?assert(is_integer(emqx_session:info(created_at, Session))). - -%%-------------------------------------------------------------------- -%% Test cases for session info/stats -%%-------------------------------------------------------------------- - -t_session_info(_) -> - ?assertMatch( - #{ - subscriptions := #{}, - upgrade_qos := false, - retry_interval := 30000, - await_rel_timeout := 300000 - }, - emqx_session:info(session()) - ). - -t_session_stats(_) -> - Stats = emqx_session:stats(session()), - ?assertMatch( - #{ - subscriptions_max := infinity, - inflight_max := 0, - mqueue_len := 0, - mqueue_max := 1000, - mqueue_dropped := 0, - next_pkt_id := 1, - awaiting_rel_cnt := 0, - awaiting_rel_max := 100 - }, - maps:from_list(Stats) - ). - -%%-------------------------------------------------------------------- -%% Test cases for sub/unsub -%%-------------------------------------------------------------------- - -t_subscribe(_) -> - ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), - {ok, Session} = emqx_session:subscribe( - clientinfo(), <<"#">>, subopts(), session() - ), - ?assertEqual(1, emqx_session:info(subscriptions_cnt, Session)). - -t_is_subscriptions_full_false(_) -> - Session = session(#{max_subscriptions => infinity}), - ?assertNot(emqx_session:is_subscriptions_full(Session)). - -t_is_subscriptions_full_true(_) -> - ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), - Session = session(#{max_subscriptions => 1}), - ?assertNot(emqx_session:is_subscriptions_full(Session)), - {ok, Session1} = emqx_session:subscribe( - clientinfo(), <<"t1">>, subopts(), Session - ), - ?assert(emqx_session:is_subscriptions_full(Session1)), - {error, ?RC_QUOTA_EXCEEDED} = - emqx_session:subscribe(clientinfo(), <<"t2">>, subopts(), Session1). - -t_unsubscribe(_) -> - ok = meck:expect(emqx_broker, unsubscribe, fun(_) -> ok end), - Session = session(#{subscriptions => #{<<"#">> => subopts()}}), - {ok, Session1} = emqx_session:unsubscribe(clientinfo(), <<"#">>, #{}, Session), - {error, ?RC_NO_SUBSCRIPTION_EXISTED} = - emqx_session:unsubscribe(clientinfo(), <<"#">>, #{}, Session1). - -t_publish_qos0(_) -> - ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), - Msg = emqx_message:make(clientid, ?QOS_0, <<"t">>, <<"payload">>), - {ok, [], Session} = emqx_session:publish(clientinfo(), 1, Msg, Session = session()), - {ok, [], Session} = emqx_session:publish(clientinfo(), undefined, Msg, Session). - -t_publish_qos1(_) -> - ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), - Msg = emqx_message:make(clientid, ?QOS_1, <<"t">>, <<"payload">>), - {ok, [], Session} = emqx_session:publish(clientinfo(), 1, Msg, Session = session()), - {ok, [], Session} = emqx_session:publish(clientinfo(), 2, Msg, Session). - -t_publish_qos2(_) -> - ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), - Msg = emqx_message:make(clientid, ?QOS_2, <<"t">>, <<"payload">>), - {ok, [], Session} = emqx_session:publish(clientinfo(), 1, Msg, session()), - ?assertEqual(1, emqx_session:info(awaiting_rel_cnt, Session)), - {ok, Session1} = emqx_session:pubrel(clientinfo(), 1, Session), - ?assertEqual(0, emqx_session:info(awaiting_rel_cnt, Session1)), - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrel(clientinfo(), 1, Session1). - -t_publish_qos2_with_error_return(_) -> - ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), - ok = meck:expect(emqx_hooks, run, fun - ('message.dropped', [Msg, _By, ReasonName]) -> - self() ! {'message.dropped', ReasonName, Msg}, - ok; - (_Hook, _Arg) -> - ok - end), - - Session = session(#{max_awaiting_rel => 2, awaiting_rel => #{PacketId1 = 1 => ts(millisecond)}}), - begin - Msg1 = emqx_message:make(clientid, ?QOS_2, <<"t">>, <<"payload1">>), - {error, RC1 = ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:publish( - clientinfo(), PacketId1, Msg1, Session - ), - receive - {'message.dropped', Reason1, RecMsg1} -> - ?assertEqual(Reason1, emqx_reason_codes:name(RC1)), - ?assertEqual(RecMsg1, Msg1) - after 1000 -> - ct:fail(?FUNCTION_NAME) - end - end, - - begin - Msg2 = emqx_message:make(clientid, ?QOS_2, <<"t">>, <<"payload2">>), - {ok, [], Session1} = emqx_session:publish(clientinfo(), _PacketId2 = 2, Msg2, Session), - ?assertEqual(2, emqx_session:info(awaiting_rel_cnt, Session1)), - {error, RC2 = ?RC_RECEIVE_MAXIMUM_EXCEEDED} = emqx_session:publish( - clientinfo(), _PacketId3 = 3, Msg2, Session1 - ), - receive - {'message.dropped', Reason2, RecMsg2} -> - ?assertEqual(Reason2, emqx_reason_codes:name(RC2)), - ?assertEqual(RecMsg2, Msg2) - after 1000 -> - ct:fail(?FUNCTION_NAME) - end - end, - ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end). - -t_is_awaiting_full_false(_) -> - Session = session(#{max_awaiting_rel => infinity}), - ?assertNot(emqx_session:is_awaiting_full(Session)). - -t_is_awaiting_full_true(_) -> - Session = session(#{ - max_awaiting_rel => 1, - awaiting_rel => #{1 => ts(millisecond)} - }), - ?assert(emqx_session:is_awaiting_full(Session)). - -t_puback(_) -> - Msg = emqx_message:make(test, ?QOS_1, <<"t">>, <<>>), - Inflight = emqx_inflight:insert(1, with_ts(wait_ack, Msg), emqx_inflight:new()), - Session = session(#{inflight => Inflight, mqueue => mqueue()}), - {ok, Msg, Session1} = emqx_session:puback(clientinfo(), 1, Session), - ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)). - -t_puback_with_dequeue(_) -> - Msg1 = emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload1">>), - Inflight = emqx_inflight:insert(1, with_ts(wait_ack, Msg1), emqx_inflight:new()), - Msg2 = emqx_message:make(clientid, ?QOS_1, <<"t2">>, <<"payload2">>), - {_, Q} = emqx_mqueue:in(Msg2, mqueue(#{max_len => 10})), - Session = session(#{inflight => Inflight, mqueue => Q}), - {ok, Msg1, [{_, Msg3}], Session1} = emqx_session:puback(clientinfo(), 1, Session), - ?assertEqual(1, emqx_session:info(inflight_cnt, Session1)), - ?assertEqual(0, emqx_session:info(mqueue_len, Session1)), - ?assertEqual(<<"t2">>, emqx_message:topic(Msg3)). - -t_puback_error_packet_id_in_use(_) -> - Inflight = emqx_inflight:insert(1, with_ts(wait_comp, undefined), emqx_inflight:new()), - {error, ?RC_PACKET_IDENTIFIER_IN_USE} = - emqx_session:puback(clientinfo(), 1, session(#{inflight => Inflight})). - -t_puback_error_packet_id_not_found(_) -> - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:puback(clientinfo(), 1, session()). - -t_pubrec(_) -> - Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<>>), - Inflight = emqx_inflight:insert(2, with_ts(wait_ack, Msg), emqx_inflight:new()), - Session = session(#{inflight => Inflight}), - {ok, Msg, Session1} = emqx_session:pubrec(clientinfo(), 2, Session), - ?assertMatch( - [#inflight_data{phase = wait_comp}], - emqx_inflight:values(emqx_session:info(inflight, Session1)) - ). - -t_pubrec_packet_id_in_use_error(_) -> - Inflight = emqx_inflight:insert(1, with_ts(wait_comp, undefined), emqx_inflight:new()), - {error, ?RC_PACKET_IDENTIFIER_IN_USE} = - emqx_session:pubrec(clientinfo(), 1, session(#{inflight => Inflight})). - -t_pubrec_packet_id_not_found_error(_) -> - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrec(clientinfo(), 1, session()). - -t_pubrel(_) -> - Session = session(#{awaiting_rel => #{1 => ts(millisecond)}}), - {ok, Session1} = emqx_session:pubrel(clientinfo(), 1, Session), - ?assertEqual(#{}, emqx_session:info(awaiting_rel, Session1)). - -t_pubrel_error_packetid_not_found(_) -> - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrel(clientinfo(), 1, session()). - -t_pubcomp(_) -> - Inflight = emqx_inflight:insert(1, with_ts(wait_comp, undefined), emqx_inflight:new()), - Session = session(#{inflight => Inflight}), - {ok, Session1} = emqx_session:pubcomp(clientinfo(), 1, Session), - ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)). - -t_pubcomp_error_packetid_in_use(_) -> - Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<>>), - Inflight = emqx_inflight:insert(1, {Msg, ts(millisecond)}, emqx_inflight:new()), - Session = session(#{inflight => Inflight}), - {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:pubcomp(clientinfo(), 1, Session). - -t_pubcomp_error_packetid_not_found(_) -> - {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubcomp(clientinfo(), 1, session()). - -%%-------------------------------------------------------------------- -%% Test cases for deliver/retry -%%-------------------------------------------------------------------- - -t_dequeue(_) -> - Q = mqueue(#{store_qos0 => true}), - {ok, Session} = emqx_session:dequeue(clientinfo(), session(#{mqueue => Q})), - Msgs = [ - emqx_message:make(clientid, ?QOS_0, <<"t0">>, <<"payload">>), - emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload">>), - emqx_message:make(clientid, ?QOS_2, <<"t2">>, <<"payload">>) - ], - Session1 = lists:foldl( - fun(Msg, S) -> - emqx_session:enqueue(clientinfo(), Msg, S) - end, - Session, - Msgs - ), - {ok, [{undefined, Msg0}, {1, Msg1}, {2, Msg2}], Session2} = - emqx_session:dequeue(clientinfo(), Session1), - ?assertEqual(0, emqx_session:info(mqueue_len, Session2)), - ?assertEqual(2, emqx_session:info(inflight_cnt, Session2)), - ?assertEqual(<<"t0">>, emqx_message:topic(Msg0)), - ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)), - ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)). - -t_deliver_qos0(_) -> - ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), - {ok, Session} = emqx_session:subscribe( - clientinfo(), <<"t0">>, subopts(), session() - ), - {ok, Session1} = emqx_session:subscribe( - clientinfo(), <<"t1">>, subopts(), Session - ), - Deliveries = [delivery(?QOS_0, T) || T <- [<<"t0">>, <<"t1">>]], - {ok, [{undefined, Msg1}, {undefined, Msg2}], Session1} = - emqx_session:deliver(clientinfo(), Deliveries, Session1), - ?assertEqual(<<"t0">>, emqx_message:topic(Msg1)), - ?assertEqual(<<"t1">>, emqx_message:topic(Msg2)). - -t_deliver_qos1(_) -> - ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), - {ok, Session} = emqx_session:subscribe( - clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session() - ), - Delivers = [delivery(?QOS_1, T) || T <- [<<"t1">>, <<"t2">>]], - {ok, [{1, Msg1}, {2, Msg2}], Session1} = emqx_session:deliver(clientinfo(), Delivers, Session), - ?assertEqual(2, emqx_session:info(inflight_cnt, Session1)), - ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)), - ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)), - {ok, Msg1T, Session2} = emqx_session:puback(clientinfo(), 1, Session1), - ?assertEqual(Msg1, remove_deliver_flag(Msg1T)), - ?assertEqual(1, emqx_session:info(inflight_cnt, Session2)), - {ok, Msg2T, Session3} = emqx_session:puback(clientinfo(), 2, Session2), - ?assertEqual(Msg2, remove_deliver_flag(Msg2T)), - ?assertEqual(0, emqx_session:info(inflight_cnt, Session3)). - -t_deliver_qos2(_) -> - ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), - Delivers = [delivery(?QOS_2, <<"t0">>), delivery(?QOS_2, <<"t1">>)], - {ok, [{1, Msg1}, {2, Msg2}], Session} = - emqx_session:deliver(clientinfo(), Delivers, session()), - ?assertEqual(2, emqx_session:info(inflight_cnt, Session)), - ?assertEqual(<<"t0">>, emqx_message:topic(Msg1)), - ?assertEqual(<<"t1">>, emqx_message:topic(Msg2)). - -t_deliver_one_msg(_) -> - {ok, [{1, Msg}], Session} = - emqx_session:deliver(clientinfo(), [delivery(?QOS_1, <<"t1">>)], session()), - ?assertEqual(1, emqx_session:info(inflight_cnt, Session)), - ?assertEqual(<<"t1">>, emqx_message:topic(Msg)). - -t_deliver_when_inflight_is_full(_) -> - Delivers = [delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)], - Session = session(#{inflight => emqx_inflight:new(1)}), - {ok, Publishes, Session1} = emqx_session:deliver(clientinfo(), Delivers, Session), - ?assertEqual(1, length(Publishes)), - ?assertEqual(1, emqx_session:info(inflight_cnt, Session1)), - ?assertEqual(1, emqx_session:info(mqueue_len, Session1)), - {ok, Msg1, [{2, Msg2}], Session2} = emqx_session:puback(clientinfo(), 1, Session1), - ?assertEqual(1, emqx_session:info(inflight_cnt, Session2)), - ?assertEqual(0, emqx_session:info(mqueue_len, Session2)), - ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)), - ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)). - -t_enqueue(_) -> - %% store_qos0 = true - Session = emqx_session:enqueue(clientinfo(), [delivery(?QOS_0, <<"t0">>)], session()), - Session1 = emqx_session:enqueue( - clientinfo(), - [ - delivery(?QOS_1, <<"t1">>), - delivery(?QOS_2, <<"t2">>) - ], - Session - ), - ?assertEqual(3, emqx_session:info(mqueue_len, Session1)). - -t_retry(_) -> - Delivers = [delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)], - %% 0.1s - RetryIntervalMs = 100, - Session = session(#{retry_interval => RetryIntervalMs}), - {ok, Pubs, Session1} = emqx_session:deliver(clientinfo(), Delivers, Session), - %% 0.2s - ElapseMs = 200, - ok = timer:sleep(ElapseMs), - Msgs1 = [{I, with_ts(wait_ack, emqx_message:set_flag(dup, Msg))} || {I, Msg} <- Pubs], - {ok, Msgs1T, 100, Session2} = emqx_session:retry(clientinfo(), Session1), - ?assertEqual(inflight_data_to_msg(Msgs1), remove_deliver_flag(Msgs1T)), - ?assertEqual(2, emqx_session:info(inflight_cnt, Session2)). - -%%-------------------------------------------------------------------- -%% Test cases for takeover/resume -%%-------------------------------------------------------------------- - -t_takeover(_) -> - ok = meck:expect(emqx_broker, unsubscribe, fun(_) -> ok end), - Session = session(#{subscriptions => #{<<"t">> => ?DEFAULT_SUBOPTS}}), - ok = emqx_session:takeover(Session). - -t_resume(_) -> - ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), - Session = session(#{subscriptions => #{<<"t">> => ?DEFAULT_SUBOPTS}}), - ok = emqx_session:resume(#{clientid => <<"clientid">>}, Session). - -t_replay(_) -> - Delivers = [delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)], - {ok, Pubs, Session1} = emqx_session:deliver(clientinfo(), Delivers, session()), - Msg = emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload">>), - Session2 = emqx_session:enqueue(clientinfo(), Msg, Session1), - Pubs1 = [{I, emqx_message:set_flag(dup, M)} || {I, M} <- Pubs], - {ok, ReplayPubs, Session3} = emqx_session:replay(clientinfo(), Session2), - ?assertEqual(Pubs1 ++ [{3, Msg}], remove_deliver_flag(ReplayPubs)), - ?assertEqual(3, emqx_session:info(inflight_cnt, Session3)). - -t_expire_awaiting_rel(_) -> - {ok, Session} = emqx_session:expire(clientinfo(), awaiting_rel, session()), - Timeout = emqx_session:info(await_rel_timeout, Session), - Session1 = emqx_session:set_field(awaiting_rel, #{1 => Ts = ts(millisecond)}, Session), - {ok, Timeout, Session2} = emqx_session:expire(clientinfo(), awaiting_rel, Session1), - ?assertEqual(#{1 => Ts}, emqx_session:info(awaiting_rel, Session2)). - -t_expire_awaiting_rel_all(_) -> - Session = session(#{awaiting_rel => #{1 => 1, 2 => 2}}), - {ok, Session1} = emqx_session:expire(clientinfo(), awaiting_rel, Session), - ?assertEqual(#{}, emqx_session:info(awaiting_rel, Session1)). - -%%-------------------------------------------------------------------- -%% CT for utility functions -%%-------------------------------------------------------------------- - -t_next_pakt_id(_) -> - Session = session(#{next_pkt_id => 16#FFFF}), - Session1 = emqx_session:next_pkt_id(Session), - ?assertEqual(1, emqx_session:info(next_pkt_id, Session1)), - Session2 = emqx_session:next_pkt_id(Session1), - ?assertEqual(2, emqx_session:info(next_pkt_id, Session2)). - -t_obtain_next_pkt_id(_) -> - Session = session(#{next_pkt_id => 16#FFFF}), - {16#FFFF, Session1} = emqx_session:obtain_next_pkt_id(Session), - ?assertEqual(1, emqx_session:info(next_pkt_id, Session1)), - {1, Session2} = emqx_session:obtain_next_pkt_id(Session1), - ?assertEqual(2, emqx_session:info(next_pkt_id, Session2)). - -%% Helper functions -%%-------------------------------------------------------------------- - -mqueue() -> mqueue(#{}). -mqueue(Opts) -> - emqx_mqueue:init(maps:merge(#{max_len => 0, store_qos0 => false}, Opts)). - -session() -> session(#{}). -session(InitFields) when is_map(InitFields) -> - Conf = emqx_cm:get_session_confs( - #{zone => default, clientid => <<"fake-test">>}, #{ - receive_maximum => 0, expiry_interval => 0 - } - ), - Session = emqx_session:init(Conf), - maps:fold( - fun(Field, Value, SessionAcc) -> - emqx_session:set_field(Field, Value, SessionAcc) - end, - Session, - InitFields - ). - -clientinfo() -> clientinfo(#{}). -clientinfo(Init) -> - maps:merge( - #{ - clientid => <<"clientid">>, - username => <<"username">> - }, - Init - ). - -subopts() -> subopts(#{}). -subopts(Init) -> - maps:merge(?DEFAULT_SUBOPTS, Init). - -delivery(QoS, Topic) -> - {deliver, Topic, emqx_message:make(test, QoS, Topic, <<"payload">>)}. - -ts(second) -> - erlang:system_time(second); -ts(millisecond) -> - erlang:system_time(millisecond). - -with_ts(Phase, Msg) -> - with_ts(Phase, Msg, erlang:system_time(millisecond)). - -with_ts(Phase, Msg, Ts) -> - #inflight_data{ - phase = Phase, - message = Msg, - timestamp = Ts - }. - -remove_deliver_flag({Id, Data}) -> - {Id, remove_deliver_flag(Data)}; -remove_deliver_flag(#inflight_data{message = Msg} = Data) -> - Data#inflight_data{message = remove_deliver_flag(Msg)}; -remove_deliver_flag(List) when is_list(List) -> - lists:map(fun remove_deliver_flag/1, List); -remove_deliver_flag(Msg) -> - emqx_message:remove_header(deliver_begin_at, Msg). - -inflight_data_to_msg({Id, Data}) -> - {Id, inflight_data_to_msg(Data)}; -inflight_data_to_msg(#inflight_data{message = Msg}) -> - Msg; -inflight_data_to_msg(List) when is_list(List) -> - lists:map(fun inflight_data_to_msg/1, List). diff --git a/apps/emqx/test/emqx_session_mem_SUITE.erl b/apps/emqx/test/emqx_session_mem_SUITE.erl new file mode 100644 index 000000000..7f10635c1 --- /dev/null +++ b/apps/emqx/test/emqx_session_mem_SUITE.erl @@ -0,0 +1,599 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2018-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_session_mem_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +-type inflight_data_phase() :: wait_ack | wait_comp. + +-record(inflight_data, { + phase :: inflight_data_phase(), + message :: emqx_types:message(), + timestamp :: non_neg_integer() +}). + +%%-------------------------------------------------------------------- +%% CT callbacks +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + ok = meck:new( + [emqx_broker, emqx_hooks, emqx_session], + [passthrough, no_history, no_link] + ), + ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end), + Apps = emqx_cth_suite:start( + [ + {emqx, #{ + override_env => [{boot_modules, [broker]}] + }} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{suite_apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(suite_apps, Config)), + meck:unload([emqx_broker, emqx_hooks]). + +%%-------------------------------------------------------------------- +%% Test cases for session init +%%-------------------------------------------------------------------- + +t_session_init(_) -> + ClientInfo = #{zone => default, clientid => <<"fake-test">>}, + ConnInfo = #{receive_maximum => 64, expiry_interval => 0}, + Session = emqx_session_mem:create( + ClientInfo, + ConnInfo, + emqx_session:get_session_conf(ClientInfo, ConnInfo) + ), + ?assertEqual(#{}, emqx_session_mem:info(subscriptions, Session)), + ?assertEqual(0, emqx_session_mem:info(subscriptions_cnt, Session)), + ?assertEqual(infinity, emqx_session_mem:info(subscriptions_max, Session)), + ?assertEqual(false, emqx_session_mem:info(upgrade_qos, Session)), + ?assertEqual(0, emqx_session_mem:info(inflight_cnt, Session)), + ?assertEqual(64, emqx_session_mem:info(inflight_max, Session)), + ?assertEqual(1, emqx_session_mem:info(next_pkt_id, Session)), + ?assertEqual(30000, emqx_session_mem:info(retry_interval, Session)), + ?assertEqual(0, emqx_mqueue:len(emqx_session_mem:info(mqueue, Session))), + ?assertEqual(0, emqx_session_mem:info(awaiting_rel_cnt, Session)), + ?assertEqual(100, emqx_session_mem:info(awaiting_rel_max, Session)), + ?assertEqual(300000, emqx_session_mem:info(await_rel_timeout, Session)), + ?assert(is_integer(emqx_session_mem:info(created_at, Session))). + +%%-------------------------------------------------------------------- +%% Test cases for session info/stats +%%-------------------------------------------------------------------- + +t_session_info(_) -> + Keys = [subscriptions, upgrade_qos, retry_interval, await_rel_timeout], + ?assertMatch( + #{ + subscriptions := #{}, + upgrade_qos := false, + retry_interval := 30000, + await_rel_timeout := 300000 + }, + maps:from_list(emqx_session_mem:info(Keys, session())) + ). + +t_session_stats(_) -> + Stats = emqx_session_mem:stats(session()), + ?assertMatch( + #{ + subscriptions_max := infinity, + inflight_max := 0, + mqueue_len := 0, + mqueue_max := 1000, + mqueue_dropped := 0, + next_pkt_id := 1, + awaiting_rel_cnt := 0, + awaiting_rel_max := 100 + }, + maps:from_list(Stats) + ). + +%%-------------------------------------------------------------------- +%% Test cases for sub/unsub +%%-------------------------------------------------------------------- + +t_subscribe(_) -> + ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), + {ok, Session} = emqx_session_mem:subscribe(<<"#">>, subopts(), session()), + ?assertEqual(1, emqx_session_mem:info(subscriptions_cnt, Session)). + +t_is_subscriptions_full_false(_) -> + Session = session(#{max_subscriptions => infinity}), + ?assertNot(emqx_session_mem:is_subscriptions_full(Session)). + +t_is_subscriptions_full_true(_) -> + ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), + Session = session(#{max_subscriptions => 1}), + ?assertNot(emqx_session_mem:is_subscriptions_full(Session)), + {ok, Session1} = emqx_session_mem:subscribe( + <<"t1">>, subopts(), Session + ), + ?assert(emqx_session_mem:is_subscriptions_full(Session1)), + {error, ?RC_QUOTA_EXCEEDED} = emqx_session_mem:subscribe( + <<"t2">>, subopts(), Session1 + ). + +t_unsubscribe(_) -> + ok = meck:expect(emqx_broker, unsubscribe, fun(_) -> ok end), + SubOpts = subopts(), + Session = session(#{subscriptions => #{<<"#">> => SubOpts}}), + {ok, Session1, SubOpts} = emqx_session_mem:unsubscribe(<<"#">>, Session), + {error, ?RC_NO_SUBSCRIPTION_EXISTED} = emqx_session_mem:unsubscribe(<<"#">>, Session1). + +t_publish_qos0(_) -> + ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), + Msg = emqx_message:make(clientid, ?QOS_0, <<"t">>, <<"payload">>), + {ok, [], [], Session} = emqx_session_mem:publish(1, Msg, Session = session()), + {ok, [], [], Session} = emqx_session_mem:publish(undefined, Msg, Session). + +t_publish_qos1(_) -> + ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), + Msg = emqx_message:make(clientid, ?QOS_1, <<"t">>, <<"payload">>), + {ok, [], [], Session} = emqx_session_mem:publish(1, Msg, Session = session()), + {ok, [], [], Session} = emqx_session_mem:publish(2, Msg, Session). + +t_publish_qos2(_) -> + ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), + Msg = emqx_message:make(clientid, ?QOS_2, <<"t">>, <<"payload">>), + {ok, [], Session} = emqx_session_mem:publish(1, Msg, session()), + ?assertEqual(1, emqx_session_mem:info(awaiting_rel_cnt, Session)), + {ok, Session1} = emqx_session_mem:pubrel(1, Session), + ?assertEqual(0, emqx_session_mem:info(awaiting_rel_cnt, Session1)), + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session_mem:pubrel(1, Session1). + +t_publish_qos2_with_error_return(_) -> + ok = meck:expect(emqx_broker, publish, fun(_) -> [] end), + ok = meck:expect(emqx_hooks, run, fun + ('message.dropped', [Msg, _By, ReasonName]) -> + self() ! {'message.dropped', ReasonName, Msg}, + ok; + (_Hook, _Arg) -> + ok + end), + + PacketId1 = 1, + Session = session(#{max_awaiting_rel => 2, awaiting_rel => #{PacketId1 => ts(millisecond)}}), + Msg1 = emqx_message:make(clientid, ?QOS_2, <<"t">>, <<"payload1">>), + {error, RC1 = ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:publish( + clientinfo(), PacketId1, Msg1, Session + ), + receive + {'message.dropped', Reason1, RecMsg1} -> + ?assertEqual(Reason1, emqx_reason_codes:name(RC1)), + ?assertEqual(RecMsg1, Msg1) + after 1000 -> + ct:fail(?FUNCTION_NAME) + end, + + Msg2 = emqx_message:make(clientid, ?QOS_2, <<"t">>, <<"payload2">>), + {ok, [], Session1} = emqx_session:publish( + clientinfo(), _PacketId2 = 2, Msg2, Session + ), + ?assertEqual(2, emqx_session_mem:info(awaiting_rel_cnt, Session1)), + {error, RC2 = ?RC_RECEIVE_MAXIMUM_EXCEEDED} = emqx_session:publish( + clientinfo(), _PacketId3 = 3, Msg2, Session1 + ), + receive + {'message.dropped', Reason2, RecMsg2} -> + ?assertEqual(Reason2, emqx_reason_codes:name(RC2)), + ?assertEqual(RecMsg2, Msg2) + after 1000 -> + ct:fail(?FUNCTION_NAME) + end, + + ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end). + +t_is_awaiting_full_false(_) -> + Session = session(#{max_awaiting_rel => infinity}), + ?assertNot(emqx_session_mem:is_awaiting_full(Session)). + +t_is_awaiting_full_true(_) -> + Session = session(#{ + max_awaiting_rel => 1, + awaiting_rel => #{1 => ts(millisecond)} + }), + ?assert(emqx_session_mem:is_awaiting_full(Session)). + +t_puback(_) -> + Msg = emqx_message:make(test, ?QOS_1, <<"t">>, <<>>), + Inflight = emqx_inflight:insert(1, with_ts(wait_ack, Msg), emqx_inflight:new()), + Session = session(#{inflight => Inflight, mqueue => mqueue()}), + {ok, Msg, [], Session1} = emqx_session_mem:puback(clientinfo(), 1, Session), + ?assertEqual(0, emqx_session_mem:info(inflight_cnt, Session1)). + +t_puback_with_dequeue(_) -> + Msg1 = emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload1">>), + Inflight = emqx_inflight:insert(1, with_ts(wait_ack, Msg1), emqx_inflight:new()), + Msg2 = emqx_message:make(clientid, ?QOS_1, <<"t2">>, <<"payload2">>), + {_, Q} = emqx_mqueue:in(Msg2, mqueue(#{max_len => 10})), + Session = session(#{inflight => Inflight, mqueue => Q}), + {ok, Msg1, [{_, Msg3}], Session1} = emqx_session_mem:puback(clientinfo(), 1, Session), + ?assertEqual(1, emqx_session_mem:info(inflight_cnt, Session1)), + ?assertEqual(0, emqx_session_mem:info(mqueue_len, Session1)), + ?assertEqual(<<"t2">>, emqx_message:topic(Msg3)). + +t_puback_error_packet_id_in_use(_) -> + Inflight = emqx_inflight:insert(1, with_ts(wait_comp, undefined), emqx_inflight:new()), + {error, ?RC_PACKET_IDENTIFIER_IN_USE} = + emqx_session_mem:puback(clientinfo(), 1, session(#{inflight => Inflight})). + +t_puback_error_packet_id_not_found(_) -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session_mem:puback(clientinfo(), 1, session()). + +t_pubrec(_) -> + Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<>>), + Inflight = emqx_inflight:insert(2, with_ts(wait_ack, Msg), emqx_inflight:new()), + Session = session(#{inflight => Inflight}), + {ok, Msg, Session1} = emqx_session_mem:pubrec(2, Session), + ?assertMatch( + [#inflight_data{phase = wait_comp}], + emqx_inflight:values(emqx_session_mem:info(inflight, Session1)) + ). + +t_pubrec_packet_id_in_use_error(_) -> + Inflight = emqx_inflight:insert(1, with_ts(wait_comp, undefined), emqx_inflight:new()), + Session = session(#{inflight => Inflight}), + {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session_mem:pubrec(1, Session). + +t_pubrec_packet_id_not_found_error(_) -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session_mem:pubrec(1, session()). + +t_pubrel(_) -> + Session = session(#{awaiting_rel => #{1 => ts(millisecond)}}), + {ok, Session1} = emqx_session_mem:pubrel(1, Session), + ?assertEqual(#{}, emqx_session_mem:info(awaiting_rel, Session1)). + +t_pubrel_error_packetid_not_found(_) -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session_mem:pubrel(1, session()). + +t_pubcomp(_) -> + Inflight = emqx_inflight:insert(1, with_ts(wait_comp, undefined), emqx_inflight:new()), + Session = session(#{inflight => Inflight}), + {ok, undefined, [], Session1} = emqx_session_mem:pubcomp(clientinfo(), 1, Session), + ?assertEqual(0, emqx_session_mem:info(inflight_cnt, Session1)). + +t_pubcomp_error_packetid_in_use(_) -> + Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<>>), + Inflight = emqx_inflight:insert(1, {Msg, ts(millisecond)}, emqx_inflight:new()), + Session = session(#{inflight => Inflight}), + {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session_mem:pubcomp(clientinfo(), 1, Session). + +t_pubcomp_error_packetid_not_found(_) -> + {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session_mem:pubcomp(clientinfo(), 1, session()). + +%%-------------------------------------------------------------------- +%% Test cases for deliver/retry +%%-------------------------------------------------------------------- + +t_dequeue(_) -> + Q = mqueue(#{store_qos0 => true}), + {ok, [], Session} = emqx_session_mem:dequeue(clientinfo(), session(#{mqueue => Q})), + Msgs = [ + emqx_message:make(clientid, ?QOS_0, <<"t0">>, <<"payload">>), + emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload">>), + emqx_message:make(clientid, ?QOS_2, <<"t2">>, <<"payload">>) + ], + Session1 = emqx_session_mem:enqueue(clientinfo(), Msgs, Session), + {ok, [{undefined, Msg0}, {1, Msg1}, {2, Msg2}], Session2} = + emqx_session_mem:dequeue(clientinfo(), Session1), + ?assertEqual(0, emqx_session_mem:info(mqueue_len, Session2)), + ?assertEqual(2, emqx_session_mem:info(inflight_cnt, Session2)), + ?assertEqual(<<"t0">>, emqx_message:topic(Msg0)), + ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)), + ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)). + +t_deliver_qos0(_) -> + ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), + {ok, Session} = emqx_session_mem:subscribe(<<"t0">>, subopts(), session()), + {ok, Session1} = emqx_session_mem:subscribe(<<"t1">>, subopts(), Session), + Deliveries = enrich([delivery(?QOS_0, T) || T <- [<<"t0">>, <<"t1">>]], Session1), + {ok, [{undefined, Msg1}, {undefined, Msg2}], Session1} = + emqx_session_mem:deliver(clientinfo(), Deliveries, Session1), + ?assertEqual(<<"t0">>, emqx_message:topic(Msg1)), + ?assertEqual(<<"t1">>, emqx_message:topic(Msg2)). + +t_deliver_qos1(_) -> + ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), + {ok, Session} = emqx_session_mem:subscribe( + <<"t1">>, subopts(#{qos => ?QOS_1}), session() + ), + Delivers = enrich([delivery(?QOS_1, T) || T <- [<<"t1">>, <<"t2">>]], Session), + {ok, [{1, Msg1}, {2, Msg2}], Session1} = + emqx_session_mem:deliver(clientinfo(), Delivers, Session), + ?assertEqual(2, emqx_session_mem:info(inflight_cnt, Session1)), + ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)), + ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)), + {ok, Msg1T, [], Session2} = emqx_session_mem:puback(clientinfo(), 1, Session1), + ?assertEqual(Msg1, remove_deliver_flag(Msg1T)), + ?assertEqual(1, emqx_session_mem:info(inflight_cnt, Session2)), + {ok, Msg2T, [], Session3} = emqx_session_mem:puback(clientinfo(), 2, Session2), + ?assertEqual(Msg2, remove_deliver_flag(Msg2T)), + ?assertEqual(0, emqx_session_mem:info(inflight_cnt, Session3)). + +t_deliver_qos2(_) -> + ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), + Session = session(), + Delivers = enrich([delivery(?QOS_2, <<"t0">>), delivery(?QOS_2, <<"t1">>)], Session), + {ok, [{1, Msg1}, {2, Msg2}], Session1} = + emqx_session_mem:deliver(clientinfo(), Delivers, Session), + ?assertEqual(2, emqx_session_mem:info(inflight_cnt, Session1)), + ?assertEqual(<<"t0">>, emqx_message:topic(Msg1)), + ?assertEqual(<<"t1">>, emqx_message:topic(Msg2)). + +t_deliver_one_msg(_) -> + Session = session(), + {ok, [{1, Msg}], Session1} = emqx_session_mem:deliver( + clientinfo(), + enrich(delivery(?QOS_1, <<"t1">>), Session), + Session + ), + ?assertEqual(1, emqx_session_mem:info(inflight_cnt, Session1)), + ?assertEqual(<<"t1">>, emqx_message:topic(Msg)). + +t_deliver_when_inflight_is_full(_) -> + Session = session(#{inflight => emqx_inflight:new(1)}), + Delivers = enrich([delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)], Session), + {ok, Publishes, Session1} = + emqx_session_mem:deliver(clientinfo(), Delivers, Session), + ?assertEqual(1, length(Publishes)), + ?assertEqual(1, emqx_session_mem:info(inflight_cnt, Session1)), + ?assertEqual(1, emqx_session_mem:info(mqueue_len, Session1)), + {ok, Msg1, [{2, Msg2}], Session2} = + emqx_session_mem:puback(clientinfo(), 1, Session1), + ?assertEqual(1, emqx_session_mem:info(inflight_cnt, Session2)), + ?assertEqual(0, emqx_session_mem:info(mqueue_len, Session2)), + ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)), + ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)). + +t_enqueue(_) -> + Session = session(#{mqueue => mqueue(#{max_len => 3, store_qos0 => true})}), + Session1 = emqx_session_mem:enqueue( + clientinfo(), + emqx_session:enrich_delivers( + clientinfo(), + [ + delivery(?QOS_0, <<"t0">>), + delivery(?QOS_1, <<"t1">>), + delivery(?QOS_2, <<"t2">>) + ], + Session + ), + Session + ), + ?assertEqual(3, emqx_session_mem:info(mqueue_len, Session1)), + Session2 = emqx_session_mem:enqueue( + clientinfo(), + emqx_session:enrich_delivers(clientinfo(), [delivery(?QOS_1, <<"drop">>)], Session1), + Session1 + ), + ?assertEqual(3, emqx_session_mem:info(mqueue_len, Session2)). + +t_enqueue_qos0(_) -> + Session = session(#{mqueue => mqueue(#{store_qos0 => false})}), + Session1 = emqx_session_mem:enqueue( + clientinfo(), + emqx_session:enrich_delivers( + clientinfo(), + [ + delivery(?QOS_0, <<"t0">>), + delivery(?QOS_1, <<"t1">>), + delivery(?QOS_2, <<"t2">>) + ], + Session + ), + Session + ), + ?assertEqual(2, emqx_session_mem:info(mqueue_len, Session1)). + +t_retry(_) -> + RetryIntervalMs = 1000, + Session = session(#{retry_interval => RetryIntervalMs}), + Delivers = enrich( + [ + delivery(?QOS_1, <<"t1">>, <<"expiressoon">>, _Expiry = 1), + delivery(?QOS_2, <<"t2">>), + delivery(?QOS_0, <<"t3">>), + delivery(?QOS_1, <<"t4">>) + ], + Session + ), + {ok, Pubs, Session1} = emqx_session_mem:deliver(clientinfo(), Delivers, Session), + [_Pub1, Pub2, _Pub3, Pub4] = Pubs, + {ok, _Msg, Session2} = emqx_session_mem:pubrec(get_packet_id(Pub2), Session1), + ElapseMs = 1500, + ok = timer:sleep(ElapseMs), + {ok, PubsRetry, RetryIntervalMs, Session3} = emqx_session_mem:handle_timeout( + clientinfo(), retry_delivery, Session2 + ), + ?assertEqual( + [ + % Pub1 is expired + {pubrel, get_packet_id(Pub2)}, + % Pub3 is QoS0 + set_duplicate_pub(Pub4) + ], + remove_deliver_flag(PubsRetry) + ), + ?assertEqual( + 2, + emqx_session_mem:info(inflight_cnt, Session3) + ). + +%%-------------------------------------------------------------------- +%% Test cases for takeover/resume +%%-------------------------------------------------------------------- + +t_takeover(_) -> + ok = meck:expect(emqx_broker, unsubscribe, fun(_) -> ok end), + Session = session(#{subscriptions => #{<<"t">> => ?DEFAULT_SUBOPTS}}), + ok = emqx_session_mem:takeover(Session). + +t_resume(_) -> + ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end), + Session = session(#{subscriptions => #{<<"t">> => ?DEFAULT_SUBOPTS}}), + _ = emqx_session_mem:resume(#{clientid => <<"clientid">>}, Session). + +t_replay(_) -> + Session = session(), + Messages = enrich([delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)], Session), + {ok, Pubs, Session1} = emqx_session_mem:deliver(clientinfo(), Messages, Session), + Msg = emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload">>), + Session2 = emqx_session_mem:enqueue(clientinfo(), [Msg], Session1), + Pubs1 = [{I, emqx_message:set_flag(dup, M)} || {I, M} <- Pubs], + Pendings = + [Msg4, Msg5] = enrich( + [_D4 = delivery(?QOS_1, <<"t4">>), D5 = delivery(?QOS_2, <<"t5">>)], + Session1 + ), + _ = self() ! D5, + _ = self() ! D6 = delivery(?QOS_1, <<"t6">>), + [Msg6] = enrich([D6], Session1), + {ok, ReplayPubs, Session3} = emqx_session_mem:replay(clientinfo(), Pendings, Session2), + ?assertEqual( + Pubs1 ++ [{3, Msg}, {4, Msg4}, {5, Msg5}, {6, Msg6}], + remove_deliver_flag(ReplayPubs) + ), + ?assertEqual(6, emqx_session_mem:info(inflight_cnt, Session3)). + +t_expire_awaiting_rel(_) -> + Now = ts(millisecond), + AwaitRelTimeout = 10000, + Session = session(#{await_rel_timeout => AwaitRelTimeout}), + Ts1 = Now - 1000, + Ts2 = Now - 20000, + {ok, [], Session1} = emqx_session_mem:expire(clientinfo(), Session), + Session2 = emqx_session_mem:set_field(awaiting_rel, #{1 => Ts1, 2 => Ts2}, Session1), + {ok, [], Timeout, Session3} = emqx_session_mem:expire(clientinfo(), Session2), + ?assertEqual(#{1 => Ts1}, emqx_session_mem:info(awaiting_rel, Session3)), + ?assert(Timeout =< AwaitRelTimeout). + +t_expire_awaiting_rel_all(_) -> + Session = session(#{awaiting_rel => #{1 => 1, 2 => 2}}), + {ok, [], Session1} = emqx_session_mem:expire(clientinfo(), Session), + ?assertEqual(#{}, emqx_session_mem:info(awaiting_rel, Session1)). + +%%-------------------------------------------------------------------- +%% CT for utility functions +%%-------------------------------------------------------------------- + +t_next_pakt_id(_) -> + Session = session(#{next_pkt_id => 16#FFFF}), + Session1 = emqx_session_mem:next_pkt_id(Session), + ?assertEqual(1, emqx_session_mem:info(next_pkt_id, Session1)), + Session2 = emqx_session_mem:next_pkt_id(Session1), + ?assertEqual(2, emqx_session_mem:info(next_pkt_id, Session2)). + +t_obtain_next_pkt_id(_) -> + Session = session(#{next_pkt_id => 16#FFFF}), + {16#FFFF, Session1} = emqx_session_mem:obtain_next_pkt_id(Session), + ?assertEqual(1, emqx_session_mem:info(next_pkt_id, Session1)), + {1, Session2} = emqx_session_mem:obtain_next_pkt_id(Session1), + ?assertEqual(2, emqx_session_mem:info(next_pkt_id, Session2)). + +%% Helper functions +%%-------------------------------------------------------------------- + +mqueue() -> mqueue(#{}). +mqueue(Opts) -> + emqx_mqueue:init(maps:merge(#{max_len => 0, store_qos0 => false}, Opts)). + +session() -> session(#{}). +session(InitFields) when is_map(InitFields) -> + ClientInfo = #{zone => default, clientid => <<"fake-test">>}, + ConnInfo = #{receive_maximum => 0, expiry_interval => 0}, + Session = emqx_session_mem:create( + ClientInfo, + ConnInfo, + emqx_session:get_session_conf(ClientInfo, ConnInfo) + ), + maps:fold( + fun(Field, Value, SessionAcc) -> + emqx_session_mem:set_field(Field, Value, SessionAcc) + end, + Session, + InitFields + ). + +clientinfo() -> clientinfo(#{}). +clientinfo(Init) -> + maps:merge( + #{ + clientid => <<"clientid">>, + username => <<"username">> + }, + Init + ). + +subopts() -> subopts(#{}). +subopts(Init) -> + maps:merge(?DEFAULT_SUBOPTS, Init). + +delivery(QoS, Topic) -> + Payload = emqx_guid:to_hexstr(emqx_guid:gen()), + {deliver, Topic, emqx_message:make(test, QoS, Topic, Payload)}. + +delivery(QoS, Topic, Payload, ExpiryInterval) -> + Headers = #{properties => #{'Message-Expiry-Interval' => ExpiryInterval}}, + {deliver, Topic, emqx_message:make(test, QoS, Topic, Payload, #{}, Headers)}. + +enrich(Delivers, Session) when is_list(Delivers) -> + emqx_session:enrich_delivers(clientinfo(), Delivers, Session); +enrich(Delivery, Session) when is_tuple(Delivery) -> + enrich([Delivery], Session). + +ts(second) -> + erlang:system_time(second); +ts(millisecond) -> + erlang:system_time(millisecond). + +with_ts(Phase, Msg) -> + with_ts(Phase, Msg, erlang:system_time(millisecond)). + +with_ts(Phase, Msg, Ts) -> + #inflight_data{ + phase = Phase, + message = Msg, + timestamp = Ts + }. + +remove_deliver_flag({pubrel, Id}) -> + {pubrel, Id}; +remove_deliver_flag({Id, Data}) -> + {Id, remove_deliver_flag(Data)}; +remove_deliver_flag(List) when is_list(List) -> + lists:map(fun remove_deliver_flag/1, List); +remove_deliver_flag(Msg) -> + emqx_message:remove_header(deliver_begin_at, Msg). + +set_duplicate_pub({Id, Msg}) -> + {Id, emqx_message:set_flag(dup, Msg)}. + +get_packet_id({Id, _}) -> + Id. diff --git a/apps/emqx/test/emqx_takeover_SUITE.erl b/apps/emqx/test/emqx_takeover_SUITE.erl index 4ba04c758..3f86cd3f3 100644 --- a/apps/emqx/test/emqx_takeover_SUITE.erl +++ b/apps/emqx/test/emqx_takeover_SUITE.erl @@ -160,4 +160,15 @@ assert_messages_order([Msg | Ls1], [{publish, #{payload := No}} | Ls2]) -> end. messages(Cnt) -> - [emqx_message:make(ct, 1, ?TOPIC, integer_to_binary(I)) || I <- lists:seq(1, Cnt)]. + [emqx_message:make(ct, 1, ?TOPIC, payload(I)) || I <- lists:seq(1, Cnt)]. + +payload(I) -> + % NOTE + % Introduce randomness so that natural order is not the same as arrival order. + iolist_to_binary( + io_lib:format("~4.16.0B [~B] [~s]", [ + rand:uniform(16#10000) - 1, + I, + emqx_utils_calendar:now_to_rfc3339(millisecond) + ]) + ). diff --git a/apps/emqx/test/emqx_ws_connection_SUITE.erl b/apps/emqx/test/emqx_ws_connection_SUITE.erl index b31b39ce1..83224958e 100644 --- a/apps/emqx/test/emqx_ws_connection_SUITE.erl +++ b/apps/emqx/test/emqx_ws_connection_SUITE.erl @@ -611,10 +611,10 @@ channel(InitFields) -> is_superuser => false, mountpoint => undefined }, - Conf = emqx_cm:get_session_confs(ClientInfo, #{ - receive_maximum => 0, expiry_interval => 0 - }), - Session = emqx_session:init(Conf), + Session = emqx_session:create( + ClientInfo, + #{receive_maximum => 0, expiry_interval => 0} + ), maps:fold( fun(Field, Value, Channel) -> emqx_channel:set_field(Field, Value, Channel) diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index 697dd88a8..b311d2550 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -27,9 +27,10 @@ %% Session: -export([ session_open/1, + session_ensure_new/2, session_drop/1, session_suspend/1, - session_add_iterator/2, + session_add_iterator/3, session_get_iterator_id/2, session_del_iterator/2, session_stats/0 @@ -51,6 +52,7 @@ shard/0, shard_id/0, topic/0, + topic_filter/0, time/0 ]). @@ -60,6 +62,17 @@ %% Type declarations %%================================================================================ +%% Session +%% See also: `#session{}`. +-type session() :: #{ + id := emqx_ds:session_id(), + created_at := _Millisecond :: non_neg_integer(), + expires_at := _Millisecond :: non_neg_integer() | never, + props := map() +}. + +-type iterators() :: #{topic_filter() => iterator()}. + %% Currently, this is the clientid. We avoid `emqx_types:clientid()' because that can be %% an atom, in theory (?). -type session_id() :: binary(). @@ -68,17 +81,18 @@ -type iterator_id() :: binary(). -%%-type session() :: #session{}. - -type message_store_opts() :: #{}. -type message_stats() :: #{}. -type message_id() :: binary(). -%% Parsed topic: +%% Parsed topic. -type topic() :: list(binary()). +%% Parsed topic filter. +-type topic_filter() :: list(binary() | '+' | '#' | ''). + -type keyspace() :: atom(). -type shard_id() :: binary(). -type shard() :: {keyspace(), shard_id()}. @@ -92,7 +106,7 @@ -type replay_id() :: binary(). -type replay() :: { - _TopicFilter :: emqx_topic:words(), + _TopicFilter :: topic_filter(), _StartTime :: time() }. @@ -135,39 +149,59 @@ message_stats() -> %%-------------------------------------------------------------------------------- %% @doc Called when a client connects. This function looks up a -%% session or creates a new one if previous one couldn't be found. +%% session or returns `false` if previous one couldn't be found. %% %% This function also spawns replay agents for each iterator. %% %% Note: session API doesn't handle session takeovers, it's the job of %% the broker. --spec session_open(emqx_types:clientid()) -> {_New :: boolean(), session_id()}. -session_open(ClientID) -> - {atomic, Res} = - mria:transaction(?DS_MRIA_SHARD, fun() -> - case mnesia:read(?SESSION_TAB, ClientID, write) of - [#session{}] -> - {false, ClientID}; - [] -> - Session = #session{id = ClientID}, - mnesia:write(?SESSION_TAB, Session, write), - {true, ClientID} - end - end), - Res. +-spec session_open(session_id()) -> + {ok, session(), iterators()} | false. +session_open(SessionId) -> + transaction(fun() -> + case mnesia:read(?SESSION_TAB, SessionId, write) of + [Record = #session{}] -> + Session = export_record(Record), + IteratorRefs = session_read_iterators(SessionId), + Iterators = export_iterators(IteratorRefs), + {ok, Session, Iterators}; + [] -> + false + end + end). + +-spec session_ensure_new(session_id(), _Props :: map()) -> + {ok, session(), iterators()}. +session_ensure_new(SessionId, Props) -> + transaction(fun() -> + ok = session_drop_iterators(SessionId), + Session = export_record(session_create(SessionId, Props)), + {ok, Session, #{}} + end). + +session_create(SessionId, Props) -> + Session = #session{ + id = SessionId, + created_at = erlang:system_time(millisecond), + expires_at = never, + props = Props + }, + ok = mnesia:write(?SESSION_TAB, Session, write), + Session. %% @doc Called when a client reconnects with `clean session=true' or %% during session GC --spec session_drop(emqx_types:clientid()) -> ok. -session_drop(ClientID) -> - {atomic, ok} = mria:transaction( - ?DS_MRIA_SHARD, - fun() -> - %% TODO: ensure all iterators from this clientid are closed? - mnesia:delete({?SESSION_TAB, ClientID}) - end - ), - ok. +-spec session_drop(session_id()) -> ok. +session_drop(DSSessionId) -> + transaction(fun() -> + %% TODO: ensure all iterators from this clientid are closed? + ok = session_drop_iterators(DSSessionId), + ok = mnesia:delete(?SESSION_TAB, DSSessionId, write) + end). + +session_drop_iterators(DSSessionId) -> + IteratorRefs = session_read_iterators(DSSessionId), + ok = lists:foreach(fun session_del_iterator/1, IteratorRefs). %% @doc Called when a client disconnects. This function terminates all %% active processes related to the session. @@ -177,39 +211,48 @@ session_suspend(_SessionId) -> ok. %% @doc Called when a client subscribes to a topic. Idempotent. --spec session_add_iterator(session_id(), emqx_topic:words()) -> - {ok, iterator_id(), time(), _IsNew :: boolean()}. -session_add_iterator(DSSessionId, TopicFilter) -> +-spec session_add_iterator(session_id(), topic_filter(), _Props :: map()) -> + {ok, iterator(), _IsNew :: boolean()}. +session_add_iterator(DSSessionId, TopicFilter, Props) -> IteratorRefId = {DSSessionId, TopicFilter}, - {atomic, Res} = - mria:transaction(?DS_MRIA_SHARD, fun() -> - case mnesia:read(?ITERATOR_REF_TAB, IteratorRefId, write) of - [] -> - {IteratorId, StartMS} = new_iterator_id(DSSessionId), - IteratorRef = #iterator_ref{ - ref_id = IteratorRefId, - it_id = IteratorId, - start_time = StartMS - }, - ok = mnesia:write(?ITERATOR_REF_TAB, IteratorRef, write), - ?tp( - ds_session_subscription_added, - #{iterator_id => IteratorId, session_id => DSSessionId} - ), - IsNew = true, - {ok, IteratorId, StartMS, IsNew}; - [#iterator_ref{it_id = IteratorId, start_time = StartMS}] -> - ?tp( - ds_session_subscription_present, - #{iterator_id => IteratorId, session_id => DSSessionId} - ), - IsNew = false, - {ok, IteratorId, StartMS, IsNew} - end - end), - Res. + transaction(fun() -> + case mnesia:read(?ITERATOR_REF_TAB, IteratorRefId, write) of + [] -> + IteratorRef = session_insert_iterator(DSSessionId, TopicFilter, Props), + Iterator = export_record(IteratorRef), + ?tp( + ds_session_subscription_added, + #{iterator => Iterator, session_id => DSSessionId} + ), + {ok, Iterator, _IsNew = true}; + [#iterator_ref{} = IteratorRef] -> + NIteratorRef = session_update_iterator(IteratorRef, Props), + NIterator = export_record(NIteratorRef), + ?tp( + ds_session_subscription_present, + #{iterator => NIterator, session_id => DSSessionId} + ), + {ok, NIterator, _IsNew = false} + end + end). --spec session_get_iterator_id(session_id(), emqx_topic:words()) -> +session_insert_iterator(DSSessionId, TopicFilter, Props) -> + {IteratorId, StartMS} = new_iterator_id(DSSessionId), + IteratorRef = #iterator_ref{ + ref_id = {DSSessionId, TopicFilter}, + it_id = IteratorId, + start_time = StartMS, + props = Props + }, + ok = mnesia:write(?ITERATOR_REF_TAB, IteratorRef, write), + IteratorRef. + +session_update_iterator(IteratorRef, Props) -> + NIteratorRef = IteratorRef#iterator_ref{props = Props}, + ok = mnesia:write(?ITERATOR_REF_TAB, NIteratorRef, write), + NIteratorRef. + +-spec session_get_iterator_id(session_id(), topic_filter()) -> {ok, iterator_id()} | {error, not_found}. session_get_iterator_id(DSSessionId, TopicFilter) -> IteratorRefId = {DSSessionId, TopicFilter}, @@ -221,14 +264,23 @@ session_get_iterator_id(DSSessionId, TopicFilter) -> end. %% @doc Called when a client unsubscribes from a topic. --spec session_del_iterator(session_id(), emqx_topic:words()) -> ok. +-spec session_del_iterator(session_id(), topic_filter()) -> ok. session_del_iterator(DSSessionId, TopicFilter) -> IteratorRefId = {DSSessionId, TopicFilter}, - {atomic, ok} = - mria:transaction(?DS_MRIA_SHARD, fun() -> - mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write) - end), - ok. + transaction(fun() -> + mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write) + end). + +session_del_iterator(#iterator_ref{ref_id = IteratorRefId}) -> + mnesia:delete(?ITERATOR_REF_TAB, IteratorRefId, write). + +session_read_iterators(DSSessionId) -> + % NOTE: somewhat convoluted way to trick dialyzer + Pat = erlang:make_tuple(record_info(size, iterator_ref), '_', [ + {1, iterator_ref}, + {#iterator_ref.ref_id, {DSSessionId, '_'}} + ]), + mnesia:match_object(?ITERATOR_REF_TAB, Pat, read). -spec session_stats() -> #{}. session_stats() -> @@ -263,3 +315,30 @@ new_iterator_id(DSSessionId) -> NowMS = erlang:system_time(microsecond), IteratorId = <>, {IteratorId, NowMS}. + +%%-------------------------------------------------------------------------------- + +transaction(Fun) -> + {atomic, Res} = mria:transaction(?DS_MRIA_SHARD, Fun), + Res. + +%%-------------------------------------------------------------------------------- + +export_iterators(IteratorRefs) -> + lists:foldl( + fun(IteratorRef = #iterator_ref{ref_id = {_DSSessionId, TopicFilter}}, Acc) -> + Acc#{TopicFilter => export_record(IteratorRef)} + end, + #{}, + IteratorRefs + ). + +export_record(#session{} = Record) -> + export_record(Record, #session.id, [id, created_at, expires_at, props], #{}); +export_record(#iterator_ref{} = Record) -> + export_record(Record, #iterator_ref.it_id, [id, start_time, props], #{}). + +export_record(Record, I, [Field | Rest], Acc) -> + export_record(Record, I + 1, Rest, Acc#{Field => element(I, Record)}); +export_record(_, _, [], Acc) -> + Acc. diff --git a/apps/emqx_durable_storage/src/emqx_ds_int.hrl b/apps/emqx_durable_storage/src/emqx_ds_int.hrl index 28a0db429..162d14b83 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_int.hrl +++ b/apps/emqx_durable_storage/src/emqx_ds_int.hrl @@ -23,14 +23,18 @@ -record(session, { %% same as clientid id :: emqx_ds:session_id(), + %% creation time + created_at :: _Millisecond :: non_neg_integer(), + expires_at = never :: _Millisecond :: non_neg_integer() | never, %% for future usage props = #{} :: map() }). -record(iterator_ref, { - ref_id :: {emqx_ds:session_id(), emqx_topic:words()}, + ref_id :: {emqx_ds:session_id(), emqx_ds:topic_filter()}, it_id :: emqx_ds:iterator_id(), - start_time :: emqx_ds:time() + start_time :: emqx_ds:time(), + props = #{} :: map() }). -endif. diff --git a/apps/emqx_durable_storage/src/emqx_ds_message_storage_bitmask.erl b/apps/emqx_durable_storage/src/emqx_ds_message_storage_bitmask.erl index 437cc5b06..7b141b202 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_message_storage_bitmask.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_message_storage_bitmask.erl @@ -132,6 +132,7 @@ %%================================================================================ -type topic() :: emqx_ds:topic(). +-type topic_filter() :: emqx_ds:topic_filter(). -type time() :: emqx_ds:time(). %% Number of bits @@ -191,7 +192,7 @@ -record(filter, { keymapper :: keymapper(), - topic_filter :: emqx_topic:words(), + topic_filter :: topic_filter(), start_time :: integer(), hash_bitfilter :: integer(), hash_bitmask :: integer(), @@ -412,11 +413,11 @@ extract(Key, #keymapper{bitsize = Size}) -> <> = Key, Bitstring. --spec compute_bitstring(topic(), time(), keymapper()) -> integer(). -compute_bitstring(Topic, Timestamp, #keymapper{source = Source}) -> - compute_bitstring(Topic, Timestamp, Source, 0). +-spec compute_bitstring(topic_filter(), time(), keymapper()) -> integer(). +compute_bitstring(TopicFilter, Timestamp, #keymapper{source = Source}) -> + compute_bitstring(TopicFilter, Timestamp, Source, 0). --spec compute_topic_bitmask(emqx_topic:words(), keymapper()) -> integer(). +-spec compute_topic_bitmask(topic_filter(), keymapper()) -> integer(). compute_topic_bitmask(TopicFilter, #keymapper{source = Source}) -> compute_topic_bitmask(TopicFilter, Source, 0). diff --git a/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl index f6ad11167..9c4b01699 100644 --- a/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl +++ b/apps/emqx_eviction_agent/src/emqx_eviction_agent_channel.erl @@ -7,7 +7,6 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_channel.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). @@ -122,7 +121,9 @@ handle_call( pendings := Pendings } = Channel ) -> - ok = emqx_session:takeover(Session), + % NOTE + % This is essentially part of `emqx_session_mem` logic, thus call it directly. + ok = emqx_session_mem:takeover(Session), %% TODO: Should not drain deliver here (side effect) Delivers = emqx_utils:drain_deliver(), AllPendings = lists:append(Delivers, Pendings), @@ -196,8 +197,11 @@ handle_deliver( clientinfo := ClientInfo } = Channel ) -> + % NOTE + % This is essentially part of `emqx_session_mem` logic, thus call it directly. Delivers1 = emqx_channel:maybe_nack(Delivers), - NSession = emqx_session:enqueue(ClientInfo, Delivers1, Session), + Messages = emqx_session:enrich_delivers(ClientInfo, Delivers1, Session), + NSession = emqx_session_mem:enqueue(ClientInfo, Messages, Session), Channel#{session := NSession}. cancel_expiry_timer(#{expiry_timer := TRef}) when is_reference(TRef) -> @@ -230,7 +234,7 @@ open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) -> } ), {error, no_session}; - {ok, #{session := Session, present := true, pendings := Pendings0}} -> + {ok, #{session := Session, present := true, replay := Pendings}} -> ?SLOG( info, #{ @@ -239,12 +243,15 @@ open_session(ConnInfo, #{clientid := ClientId} = ClientInfo) -> node => node() } ), - Pendings1 = lists:usort(lists:append(Pendings0, emqx_utils:drain_deliver())), - NSession = emqx_session:enqueue( - ClientInfo, - emqx_channel:maybe_nack(Pendings1), - Session - ), + % NOTE + % Here we aggregate and deduplicate remote and local pending deliveries, + % throwing away any local deliveries that are part of some shared + % subscription. Remote deliviries pertaining to shared subscriptions should + % already have been thrown away by `emqx_channel:handle_deliver/2`. + % See also: `emqx_channel:maybe_resume_session/1`, `emqx_session_mem:replay/3`. + DeliversLocal = emqx_channel:maybe_nack(emqx_utils:drain_deliver()), + PendingsAll = emqx_session_mem:dedup(ClientInfo, Pendings, DeliversLocal, Session), + NSession = emqx_session_mem:enqueue(ClientInfo, PendingsAll, Session), NChannel = Channel#{session => NSession}, ok = emqx_cm:insert_channel_info(ClientId, info(NChannel), stats(NChannel)), ?SLOG( diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl index 1d77fe170..b4d7ceb08 100644 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_channel_SUITE.erl @@ -29,16 +29,6 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([emqx_eviction_agent, emqx_conf]). -init_per_testcase(t_persistence, _Config) -> - {skip, "Existing session persistence implementation is being phased out"}; -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(t_persistence, Config) -> - Config; -end_per_testcase(_TestCase, _Config) -> - ok. - %%-------------------------------------------------------------------- %% Tests %%-------------------------------------------------------------------- @@ -199,40 +189,6 @@ t_get_connected_client_count(_Config) -> emqx_cm:get_connected_client_count() ). -t_persistence(_Config) -> - erlang:process_flag(trap_exit, true), - - Topic = <<"t1">>, - Message = <<"message_to_persist">>, - - {ok, C0} = emqtt_connect(?CLIENT_ID, false), - {ok, _, _} = emqtt:subscribe(C0, Topic, 0), - - Opts = evict_session_opts(?CLIENT_ID), - {ok, Pid} = emqx_eviction_agent_channel:start_supervised(Opts), - - {ok, C1} = emqtt_connect(), - {ok, _} = emqtt:publish(C1, Topic, Message, 1), - ok = emqtt:disconnect(C1), - - %% Kill channel so that the session is only persisted - ok = emqx_eviction_agent_channel:call(Pid, kick), - - %% Should restore session from persistents storage and receive messages - {ok, C2} = emqtt_connect(?CLIENT_ID, false), - - receive - {publish, #{ - payload := Message, - topic := Topic - }} -> - ok - after 1000 -> - ct:fail("message not received") - end, - - ok = emqtt:disconnect(C2). - %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 3c9e487bc..7df3b2552 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -388,8 +388,8 @@ open_session( {ok, #{session => Session, present => false}} end, case takeover_session(GwName, ClientId) of - {ok, ConnMod, ChanPid, Session} -> - ok = SessionMod:resume(ClientInfo, Session), + {ok, ConnMod, ChanPid, SessionIn} -> + Session = SessionMod:resume(ClientInfo, SessionIn), case request_stepdown({takeover, 'end'}, ConnMod, ChanPid) of {ok, Pendings} -> register_channel( diff --git a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src index a0cbc3e18..65de725a9 100644 --- a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_coap, [ {description, "CoAP Gateway"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 80d3282c5..a1d598923 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -74,12 +74,6 @@ -type replies() :: emqx_types:packet() | reply() | [reply()]. --define(TIMER_TABLE, #{ - alive_timer => keepalive, - force_timer => force_close, - idle_timer => force_close_idle -}). - -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). %%-------------------------------------------------------------------- @@ -224,7 +218,7 @@ address({Host, Port}) -> %% avoid udp connection process leak start_idle_checking_timer(Channel = #channel{conninfo = #{socktype := udp}}) -> - ensure_timer(idle_timer, Channel); + ensure_timer(force_close_idle, Channel); start_idle_checking_timer(Channel) -> Channel. @@ -293,10 +287,10 @@ handle_timeout( case emqx_keepalive:check(StatVal, Keepalive) of {ok, NKeepalive} -> NChannel = Channel#channel{keepalive = NKeepalive}, - {ok, reset_timer(alive_timer, NChannel)}; + {ok, reset_timer(keepalive, NChannel)}; {error, timeout} -> Req = #{type => 'KEEPALIVE'}, - NChannel = remove_timer_ref(alive_timer, Channel), + NChannel = remove_timer_ref(keepalive, Channel), %% close connection if keepalive timeout Replies = [{event, disconnected}, {close, keepalive_timeout}], NChannel1 = dispatch(on_timer_timeout, Req, NChannel#channel{ @@ -419,7 +413,7 @@ handle_call( NConnInfo = ConnInfo#{keepalive => Interval}, NClientInfo = ClientInfo#{keepalive => Interval}, NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo}, - {reply, ok, [{event, updated}], ensure_keepalive(cancel_timer(idle_timer, NChannel))}; + {reply, ok, [{event, updated}], ensure_keepalive(cancel_timer(force_close_idle, NChannel))}; handle_call( {subscribe_from_client, TopicFilter, Qos}, _From, @@ -529,7 +523,7 @@ handle_info( _ -> Channel end, - Channel2 = ensure_timer(force_timer, Channel1), + Channel2 = ensure_timer(force_close, Channel1), {ok, ensure_disconnected(Reason, Channel2)} end; handle_info( @@ -547,13 +541,13 @@ handle_info( ShutdownNow = emqx_exproto_gcli:is_empty(GClient) andalso - maps:get(force_timer, Timers, undefined) =/= undefined, + maps:get(force_close, Timers, undefined) =/= undefined, case Result of ok when not ShutdownNow -> GClient1 = emqx_exproto_gcli:maybe_shoot(GClient), {ok, Channel#channel{gcli = GClient1}}; ok when ShutdownNow -> - Channel1 = cancel_timer(force_timer, Channel), + Channel1 = cancel_timer(force_close, Channel), {shutdown, Channel1#channel.closed_reason, Channel1}; {error, Reason} -> {shutdown, {error, {FunName, Reason}}, Channel} @@ -711,7 +705,7 @@ ensure_keepalive_timer(Interval, Channel) when Interval =< 0 -> ensure_keepalive_timer(Interval, Channel) -> StatVal = emqx_gateway_conn:keepalive_stats(recv), Keepalive = emqx_keepalive:init(StatVal, timer:seconds(Interval)), - ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). + ensure_timer(keepalive, Channel#channel{keepalive = Keepalive}). ensure_timer(Name, Channel = #channel{timers = Timers}) -> TRef = maps:get(Name, Timers, undefined), @@ -723,8 +717,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> end. ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> - Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_utils:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Name), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> @@ -737,11 +730,11 @@ cancel_timer(Name, Channel = #channel{timers = Timers}) -> remove_timer_ref(Name, Channel = #channel{timers = Timers}) -> Channel#channel{timers = maps:remove(Name, Timers)}. -interval(idle_timer, #channel{conninfo = #{idle_timeout := IdleTimeout}}) -> +interval(force_close_idle, #channel{conninfo = #{idle_timeout := IdleTimeout}}) -> IdleTimeout; -interval(force_timer, _) -> +interval(force_close, _) -> 15000; -interval(alive_timer, #channel{keepalive = Keepalive}) -> +interval(keepalive, #channel{keepalive = Keepalive}) -> emqx_keepalive:info(interval, Keepalive). %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src index 5959eea3d..09622763b 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_exproto, [ {description, "ExProto Gateway"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src index 5e79d4d49..11c94fb3c 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_mqttsn, [ {description, "MQTT-SN Gateway"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index 2443b149a..087187379 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -104,15 +104,6 @@ -type replies() :: reply() | [reply()]. --define(TIMER_TABLE, #{ - alive_timer => keepalive, - retry_timer => retry_delivery, - await_timer => expire_awaiting_rel, - expire_timer => expire_session, - asleep_timer => expire_asleep, - register_timer => retry_register -}). - -define(DEFAULT_OVERRIDE, #{ clientid => <<"${ConnInfo.clientid}">> %, username => <<"${ConnInfo.clientid}">> @@ -431,7 +422,7 @@ ensure_keepalive_timer(0, Channel) -> Channel; ensure_keepalive_timer(Interval, Channel) -> Keepalive = emqx_keepalive:init(round(timer:seconds(Interval))), - ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). + ensure_timer(keepalive, Channel#channel{keepalive = Keepalive}). %%-------------------------------------------------------------------- %% Handle incoming packet @@ -669,7 +660,7 @@ handle_in( topic_name => TopicName }), NChannel = cancel_timer( - register_timer, + retry_register, Channel#channel{register_inflight = undefined} ), send_next_register_or_replay_publish(TopicName, NChannel); @@ -692,7 +683,7 @@ handle_in( topic_name => TopicName }), NChannel = cancel_timer( - register_timer, + retry_register, Channel#channel{register_inflight = undefined} ), send_next_register_or_replay_publish(TopicName, NChannel) @@ -1165,7 +1156,7 @@ do_publish( case emqx_mqttsn_session:publish(ClientInfo, MsgId, Msg, Session) of {ok, _PubRes, NSession} -> NChannel1 = ensure_timer( - await_timer, + expire_awaiting_rel, Channel#channel{session = NSession} ), handle_out(pubrec, MsgId, NChannel1); @@ -1178,10 +1169,6 @@ do_publish( Channel ); {error, ?RC_RECEIVE_MAXIMUM_EXCEEDED} -> - ?SLOG(warning, #{ - msg => "dropped_the_qos2_packet_due_to_awaiting_rel_full", - msg_id => MsgId - }), ok = metrics_inc(Ctx, 'packets.publish.dropped'), handle_out(puback, {TopicId, MsgId, ?SN_RC_CONGESTION}, Channel) end. @@ -1439,18 +1426,11 @@ awake( clientid => ClientId, previous_state => ConnState }), - {ok, Publishes, Session1} = emqx_mqttsn_session:replay(ClientInfo, Session), - {NPublishes, NSession} = - case emqx_mqttsn_session:deliver(ClientInfo, [], Session1) of - {ok, Session2} -> - {Publishes, Session2}; - {ok, More, Session2} -> - {lists:append(Publishes, More), Session2} - end, - Channel1 = cancel_timer(asleep_timer, Channel), + {ok, Publishes, NSession} = emqx_mqttsn_session:replay(ClientInfo, Session), + Channel1 = cancel_timer(expire_asleep, Channel), {Replies0, NChannel0} = outgoing_deliver_and_register( do_deliver( - NPublishes, + Publishes, Channel1#channel{ conn_state = awake, session = NSession } @@ -1499,7 +1479,7 @@ asleep(Duration, Channel = #channel{conn_state = asleep}) -> msg => "update_asleep_timer", new_duration => Duration }), - ensure_asleep_timer(Duration, cancel_timer(asleep_timer, Channel)); + ensure_asleep_timer(Duration, cancel_timer(expire_asleep, Channel)); asleep(Duration, Channel = #channel{conn_state = connected}) -> ?SLOG(info, #{ msg => "goto_asleep_state", @@ -1907,7 +1887,7 @@ maybe_shutdown(Reason, Channel = #channel{conninfo = ConnInfo}) -> ?UINT_MAX -> {ok, Channel}; I when I > 0 -> - {ok, ensure_timer(expire_timer, I, Channel)}; + {ok, ensure_timer(expire_session, I, Channel)}; _ -> shutdown(Reason, Channel) end. @@ -2007,7 +1987,7 @@ handle_deliver( handle_out( publish, Publishes, - ensure_timer(retry_timer, NChannel) + ensure_timer(retry_delivery, NChannel) ); {ok, NSession} -> {ok, Channel#channel{session = NSession}} @@ -2068,7 +2048,7 @@ handle_timeout( case emqx_keepalive:check(StatVal, Keepalive) of {ok, NKeepalive} -> NChannel = Channel#channel{keepalive = NKeepalive}, - {ok, reset_timer(alive_timer, NChannel)}; + {ok, reset_timer(keepalive, NChannel)}; {error, timeout} -> handle_out(disconnect, ?SN_RC2_KEEPALIVE_TIMEOUT, Channel) end; @@ -2080,23 +2060,10 @@ handle_timeout( {ok, Channel}; handle_timeout( _TRef, - retry_delivery, + retry_delivery = TimerName, Channel = #channel{conn_state = asleep} ) -> - {ok, reset_timer(retry_timer, Channel)}; -handle_timeout( - _TRef, - retry_delivery, - Channel = #channel{session = Session, clientinfo = ClientInfo} -) -> - case emqx_mqttsn_session:retry(ClientInfo, Session) of - {ok, NSession} -> - {ok, clean_timer(retry_timer, Channel#channel{session = NSession})}; - {ok, Publishes, Timeout, NSession} -> - NChannel = Channel#channel{session = NSession}, - %% XXX: These replay messages should awaiting register acked? - handle_out(publish, Publishes, reset_timer(retry_timer, Timeout, NChannel)) - end; + {ok, reset_timer(TimerName, Channel)}; handle_timeout( _TRef, expire_awaiting_rel, @@ -2105,20 +2072,23 @@ handle_timeout( {ok, Channel}; handle_timeout( _TRef, - expire_awaiting_rel, + expire_awaiting_rel = TimerName, Channel = #channel{conn_state = asleep} ) -> - {ok, reset_timer(await_timer, Channel)}; + {ok, reset_timer(TimerName, Channel)}; handle_timeout( _TRef, - expire_awaiting_rel, + TimerName, Channel = #channel{session = Session, clientinfo = ClientInfo} -) -> - case emqx_mqttsn_session:expire(ClientInfo, awaiting_rel, Session) of - {ok, NSession} -> - {ok, clean_timer(await_timer, Channel#channel{session = NSession})}; - {ok, Timeout, NSession} -> - {ok, reset_timer(await_timer, Timeout, Channel#channel{session = NSession})} +) when TimerName == retry_delivery; TimerName == expire_awaiting_rel -> + case emqx_mqttsn_session:handle_timeout(ClientInfo, TimerName, Session) of + {ok, Publishes, NSession} -> + NChannel = Channel#channel{session = NSession}, + handle_out(publish, Publishes, clean_timer(TimerName, NChannel)); + {ok, Publishes, Timeout, NSession} -> + NChannel = Channel#channel{session = NSession}, + %% XXX: These replay messages should awaiting register acked? + handle_out(publish, Publishes, reset_timer(TimerName, Timeout, NChannel)) end; handle_timeout( _TRef, @@ -2210,7 +2180,7 @@ ensure_asleep_timer(Channel = #channel{asleep_timer_duration = Duration}) when ensure_asleep_timer(Durtion, Channel) -> ensure_timer( - asleep_timer, + expire_asleep, timer:seconds(Durtion), Channel#channel{asleep_timer_duration = Durtion} ). @@ -2219,9 +2189,8 @@ ensure_register_timer(Channel) -> ensure_register_timer(0, Channel). ensure_register_timer(RetryTimes, Channel = #channel{timers = Timers}) -> - Msg = maps:get(register_timer, ?TIMER_TABLE), - TRef = emqx_utils:start_timer(?REGISTER_TIMEOUT, {Msg, RetryTimes}), - Channel#channel{timers = Timers#{register_timer => TRef}}. + TRef = emqx_utils:start_timer(?REGISTER_TIMEOUT, {retry_register, RetryTimes}), + Channel#channel{timers = Timers#{retry_register => TRef}}. cancel_timer(Name, Channel = #channel{timers = Timers}) -> case maps:get(Name, Timers, undefined) of @@ -2242,8 +2211,7 @@ ensure_timer(Name, Channel = #channel{timers = Timers}) -> end. ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> - Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_utils:start_timer(Time, Msg), + TRef = emqx_utils:start_timer(Time, Name), Channel#channel{timers = Timers#{Name => TRef}}. reset_timer(Name, Channel) -> @@ -2255,13 +2223,12 @@ reset_timer(Name, Time, Channel) -> clean_timer(Name, Channel = #channel{timers = Timers}) -> Channel#channel{timers = maps:remove(Name, Timers)}. -interval(alive_timer, #channel{keepalive = KeepAlive}) -> +interval(keepalive, #channel{keepalive = KeepAlive}) -> emqx_keepalive:info(interval, KeepAlive); -interval(retry_timer, #channel{session = Session}) -> +interval(retry_delivery, #channel{session = Session}) -> emqx_mqttsn_session:info(retry_interval, Session); -interval(await_timer, #channel{session = Session}) -> +interval(expire_awaiting_rel, #channel{session = Session}) -> emqx_mqttsn_session:info(await_rel_timeout, Session). - %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_session.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_session.erl index 7c62800cc..3621aa627 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_session.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_session.erl @@ -22,8 +22,7 @@ init/1, info/1, info/2, - stats/1, - resume/2 + stats/1 ]). -export([ @@ -39,11 +38,11 @@ -export([ replay/2, deliver/3, + handle_timeout/3, obtain_next_pkt_id/1, takeover/1, - enqueue/3, - retry/2, - expire/3 + resume/2, + enqueue/3 ]). -type session() :: #{ @@ -54,12 +53,11 @@ -export_type([session/0]). init(ClientInfo) -> - Conf = emqx_cm:get_session_confs( - ClientInfo, #{receive_maximum => 1, expiry_interval => 0} - ), + ConnInfo = #{receive_maximum => 1, expiry_interval => 0}, + SessionConf = emqx_session:get_session_conf(ClientInfo, ConnInfo), #{ registry => emqx_mqttsn_registry:init(), - session => emqx_session:init(Conf) + session => emqx_session_mem:create(ClientInfo, ConnInfo, SessionConf) }. registry(#{registry := Registry}) -> @@ -98,47 +96,45 @@ subscribe(ClientInfo, Topic, SubOpts, Session) -> unsubscribe(ClientInfo, Topic, SubOpts, Session) -> with_sess(?FUNCTION_NAME, [ClientInfo, Topic, SubOpts], Session). -replay(ClientInfo, Session) -> - with_sess(?FUNCTION_NAME, [ClientInfo], Session). +deliver(ClientInfo, Delivers, Session) -> + with_sess(?FUNCTION_NAME, [ClientInfo, Delivers], Session). -deliver(ClientInfo, Delivers, Session1) -> - with_sess(?FUNCTION_NAME, [ClientInfo, Delivers], Session1). +handle_timeout(ClientInfo, Name, Session) -> + with_sess(?FUNCTION_NAME, [ClientInfo, Name], Session). obtain_next_pkt_id(Session = #{session := Sess}) -> - {Id, Sess1} = emqx_session:obtain_next_pkt_id(Sess), + {Id, Sess1} = emqx_session_mem:obtain_next_pkt_id(Sess), {Id, Session#{session := Sess1}}. takeover(_Session = #{session := Sess}) -> - emqx_session:takeover(Sess). + emqx_session_mem:takeover(Sess). + +resume(ClientInfo, Session = #{session := Sess}) -> + Session#{session := emqx_session_mem:resume(ClientInfo, Sess)}. + +replay(ClientInfo, Session = #{session := Sess}) -> + {ok, Replies, NSess} = emqx_session_mem:replay(ClientInfo, Sess), + {ok, Replies, Session#{session := NSess}}. enqueue(ClientInfo, Delivers, Session = #{session := Sess}) -> - Sess1 = emqx_session:enqueue(ClientInfo, Delivers, Sess), - Session#{session := Sess1}. - -retry(ClientInfo, Session) -> - with_sess(?FUNCTION_NAME, [ClientInfo], Session). - -expire(ClientInfo, awaiting_rel, Session) -> - with_sess(?FUNCTION_NAME, [ClientInfo, awaiting_rel], Session). - -resume(ClientInfo, #{session := Sess}) -> - emqx_session:resume(ClientInfo, Sess). + Msgs = emqx_session:enrich_delivers(ClientInfo, Delivers, Sess), + Session#{session := emqx_session_mem:enqueue(ClientInfo, Msgs, Sess)}. %%-------------------------------------------------------------------- %% internal funcs with_sess(Fun, Args, Session = #{session := Sess}) -> case apply(emqx_session, Fun, Args ++ [Sess]) of - %% for subscribe - {error, Reason} -> - {error, Reason}; - %% for pubrel + %% for subscribe / unsubscribe / pubrel {ok, Sess1} -> {ok, Session#{session := Sess1}}; - %% for publish and puback - {ok, Result, Sess1} -> - {ok, Result, Session#{session := Sess1}}; - %% for puback + %% for publish / pubrec / pubcomp / deliver + {ok, ResultReplies, Sess1} -> + {ok, ResultReplies, Session#{session := Sess1}}; + %% for puback / handle_timeout {ok, Msgs, Replies, Sess1} -> - {ok, Msgs, Replies, Session#{session := Sess1}} + {ok, Msgs, Replies, Session#{session := Sess1}}; + %% for any errors + {error, Reason} -> + {error, Reason} end. diff --git a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl index a0afd90c1..1b5443451 100644 --- a/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl @@ -40,11 +40,6 @@ -define(HOST, {127, 0, 0, 1}). -define(PORT, 1884). --define(FLAG_DUP(X), X). --define(FLAG_QOS(X), X). --define(FLAG_RETAIN(X), X). --define(FLAG_SESSION(X), X). - -define(LOG(Format, Args), ct:log("TEST: " ++ Format, Args)). -define(MAX_PRED_TOPIC_ID, ?SN_MAX_PREDEF_TOPIC_ID). @@ -1381,14 +1376,14 @@ t_asleep_test01_timeout(_) -> t_asleep_test02_to_awake_and_back(_) -> QoS = 1, - Keepalive_Duration = 1, + KeepaliveDuration = 1, SleepDuration = 5, WillTopic = <<"dead">>, WillPayload = <<10, 11, 12, 13, 14>>, {ok, Socket} = gen_udp:open(0, [binary]), ClientId = ?CLIENTID, - send_connect_msg_with_will(Socket, Keepalive_Duration, ClientId), + send_connect_msg_with_will(Socket, KeepaliveDuration, ClientId), ?assertEqual(<<2, ?SN_WILLTOPICREQ>>, receive_response(Socket)), send_willtopic_msg(Socket, WillTopic, QoS), ?assertEqual(<<2, ?SN_WILLMSGREQ>>, receive_response(Socket)),