From 5f6bcd1ebbeff197ca923435c113df3442581c39 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 18 Aug 2021 10:40:58 +0800 Subject: [PATCH 01/20] fix(config_api): remove config APIs that already been provided by apps --- apps/emqx_management/src/emqx_mgmt_api_configs.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 2c01276ce..15849c294 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -48,12 +48,15 @@ -define(ERR_MSG(MSG), list_to_binary(io_lib:format("~p", [MSG]))). +-define(CORE_CONFS, [node, log, alarm, zones, cluster, rpc, broker, sysmon, + emqx_dashboard, emqx_management]). + api_spec() -> {config_apis() ++ [config_reset_api()], []}. config_apis() -> [config_api(ConfPath, Schema) || {ConfPath, Schema} <- - get_conf_schema(emqx_config:get([]), ?MAX_DEPTH)]. + get_conf_schema(emqx_config:get([]), ?MAX_DEPTH), is_core_conf(ConfPath)]. config_api(ConfPath, Schema) -> Path = path_join(ConfPath), @@ -193,6 +196,9 @@ path_join([P], _Sp) -> str(P); path_join([P | Path], Sp) -> str(P) ++ Sp ++ path_join(Path, Sp). +is_core_conf(Path) -> + lists:member(hd(Path), ?CORE_CONFS). + str(S) when is_list(S) -> S; str(S) when is_binary(S) -> binary_to_list(S); str(S) when is_atom(S) -> atom_to_list(S). From e8e95d39ef1615058a05f00168c7f71f96fb054e Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 18 Aug 2021 14:52:57 +0800 Subject: [PATCH 02/20] refactor(config): move emqx_config:get/get_raw to emqx:get_config/get_raw_config (#5517) --- apps/emqx/src/emqx.erl | 23 ++++++++++++++++++- apps/emqx/src/emqx_alarm.erl | 8 +++---- apps/emqx/src/emqx_broker.erl | 2 +- apps/emqx/src/emqx_cm_locker.erl | 2 +- apps/emqx/src/emqx_cm_registry.erl | 2 +- apps/emqx/src/emqx_config.erl | 12 +++++----- apps/emqx/src/emqx_connection.erl | 2 +- apps/emqx/src/emqx_flapping.erl | 2 +- apps/emqx/src/emqx_global_gc.erl | 2 +- apps/emqx/src/emqx_listeners.erl | 2 +- apps/emqx/src/emqx_os_mon.erl | 8 +++---- apps/emqx/src/emqx_plugins.erl | 2 +- apps/emqx/src/emqx_router.erl | 2 +- apps/emqx/src/emqx_rpc.erl | 2 +- apps/emqx/src/emqx_shared_sub.erl | 4 ++-- apps/emqx/src/emqx_sys.erl | 4 ++-- apps/emqx/src/emqx_sys_mon.erl | 2 +- apps/emqx/src/emqx_trie.erl | 2 +- apps/emqx/src/emqx_vm_mon.erl | 6 ++--- apps/emqx/test/emqx_mqtt_caps_SUITE.erl | 4 ++-- apps/emqx_authn/src/emqx_authn_app.erl | 2 +- apps/emqx_authz/src/emqx_authz.erl | 2 +- apps/emqx_authz/test/emqx_authz_SUITE.erl | 4 ++-- .../src/emqx_bridge_mqtt_sup.erl | 2 +- apps/emqx_dashboard/src/emqx_dashboard.erl | 2 +- .../src/emqx_dashboard_admin.erl | 2 +- .../src/emqx_dashboard_collection.erl | 4 ++-- .../src/emqx_dashboard_token.erl | 2 +- .../emqx_data_bridge/src/emqx_data_bridge.erl | 2 +- apps/emqx_exhook/src/emqx_exhook_sup.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_app.erl | 2 +- apps/emqx_management/src/emqx_mgmt.erl | 2 +- .../src/emqx_mgmt_api_configs.erl | 4 ++-- apps/emqx_management/src/emqx_mgmt_auth.erl | 2 +- apps/emqx_management/src/emqx_mgmt_http.erl | 2 +- apps/emqx_modules/src/emqx_delayed.erl | 2 +- apps/emqx_modules/src/emqx_delayed_api.erl | 2 +- apps/emqx_modules/src/emqx_event_message.erl | 4 ++-- apps/emqx_modules/src/emqx_modules_app.erl | 12 +++++----- apps/emqx_modules/src/emqx_rewrite.erl | 6 ++--- apps/emqx_modules/src/emqx_telemetry.erl | 4 ++-- apps/emqx_modules/src/emqx_topic_metrics.erl | 2 +- .../src/emqx_plugin_libs_ssl.erl | 4 ++-- .../src/emqx_prometheus_api.erl | 6 ++--- .../src/emqx_prometheus_app.erl | 4 ++-- apps/emqx_retainer/src/emqx_retainer.erl | 18 +++++++-------- apps/emqx_retainer/src/emqx_retainer_api.erl | 2 +- .../src/emqx_retainer_mnesia.erl | 4 ++-- .../src/emqx_bridge_mqtt_actions.erl | 2 +- .../emqx_rule_engine/src/emqx_rule_events.erl | 2 +- apps/emqx_statsd/src/emqx_statsd_api.erl | 6 ++--- apps/emqx_statsd/src/emqx_statsd_app.erl | 4 ++-- 52 files changed, 117 insertions(+), 96 deletions(-) diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 5df45bd15..1d4686561 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -55,7 +55,12 @@ -export([ set_debug_secret/1 ]). --export([ update_config/2 +%% Configs APIs +-export([ get_config/1 + , get_config/2 + , get_raw_config/1 + , get_raw_config/2 + , update_config/2 , update_config/3 , remove_config/1 , remove_config/2 @@ -192,6 +197,22 @@ run_hook(HookPoint, Args) -> run_fold_hook(HookPoint, Args, Acc) -> emqx_hooks:run_fold(HookPoint, Args, Acc). +-spec get_config(emqx_map_lib:config_key_path()) -> term(). +get_config(KeyPath) -> + emqx_config:get(KeyPath). + +-spec get_config(emqx_map_lib:config_key_path(), term()) -> term(). +get_config(KeyPath, Default) -> + emqx_config:get(KeyPath, Default). + +-spec get_raw_config(emqx_map_lib:config_key_path()) -> term(). +get_raw_config(KeyPath) -> + emqx_config:get_raw(KeyPath). + +-spec get_raw_config(emqx_map_lib:config_key_path(), term()) -> term(). +get_raw_config(KeyPath, Default) -> + emqx_config:get_raw(KeyPath, Default). + -spec update_config(emqx_map_lib:config_key_path(), emqx_config:update_request()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(KeyPath, UpdateReq) -> diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 44b005faa..e52328d66 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -199,7 +199,7 @@ handle_call({activate_alarm, Name, Details}, _From, State) -> message = normalize_message(Name, Details), activate_at = erlang:system_time(microsecond)}, ekka_mnesia:dirty_write(?ACTIVATED_ALARM, Alarm), - do_actions(activate, Alarm, emqx_config:get([alarm, actions])), + do_actions(activate, Alarm, emqx:get_config([alarm, actions])), {reply, ok, State} end; @@ -268,11 +268,11 @@ code_change(_OldVsn, State, _Extra) -> %%------------------------------------------------------------------------------ get_validity_period() -> - emqx_config:get([alarm, validity_period]). + emqx:get_config([alarm, validity_period]). deactivate_alarm(Details, #activated_alarm{activate_at = ActivateAt, name = Name, details = Details0, message = Msg0}) -> - SizeLimit = emqx_config:get([alarm, size_limit]), + SizeLimit = emqx:get_config([alarm, size_limit]), case SizeLimit > 0 andalso (mnesia:table_info(?DEACTIVATED_ALARM, size) >= SizeLimit) of true -> case mnesia:dirty_first(?DEACTIVATED_ALARM) of @@ -289,7 +289,7 @@ deactivate_alarm(Details, #activated_alarm{activate_at = ActivateAt, name = Name erlang:system_time(microsecond)), ekka_mnesia:dirty_write(?DEACTIVATED_ALARM, HistoryAlarm), ekka_mnesia:dirty_delete(?ACTIVATED_ALARM, Name), - do_actions(deactivate, DeActAlarm, emqx_config:get([alarm, actions])). + do_actions(deactivate, DeActAlarm, emqx:get_config([alarm, actions])). make_deactivated_alarm(ActivateAt, Name, Details, Message, DeActivateAt) -> #deactivated_alarm{ diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 1248f9980..46accb9fe 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -242,7 +242,7 @@ route(Routes, Delivery) -> do_route({To, Node}, Delivery) when Node =:= node() -> {Node, To, dispatch(To, Delivery)}; do_route({To, Node}, Delivery) when is_atom(Node) -> - {Node, To, forward(Node, To, Delivery, emqx_config:get([rpc, mode]))}; + {Node, To, forward(Node, To, Delivery, emqx:get_config([rpc, mode]))}; do_route({To, Group}, Delivery) when is_tuple(Group); is_binary(Group) -> {share, To, emqx_shared_sub:dispatch(Group, To, Delivery)}. diff --git a/apps/emqx/src/emqx_cm_locker.erl b/apps/emqx/src/emqx_cm_locker.erl index c1a85d6c9..5a336d61c 100644 --- a/apps/emqx/src/emqx_cm_locker.erl +++ b/apps/emqx/src/emqx_cm_locker.erl @@ -62,5 +62,5 @@ unlock(ClientId) -> -spec(strategy() -> local | leader | quorum | all). strategy() -> - emqx_config:get([broker, session_locking_strategy]). + emqx:get_config([broker, session_locking_strategy]). diff --git a/apps/emqx/src/emqx_cm_registry.erl b/apps/emqx/src/emqx_cm_registry.erl index c04f6ccaf..9326d2b8e 100644 --- a/apps/emqx/src/emqx_cm_registry.erl +++ b/apps/emqx/src/emqx_cm_registry.erl @@ -65,7 +65,7 @@ start_link() -> %% @doc Is the global registry enabled? -spec(is_enabled() -> boolean()). is_enabled() -> - emqx_config:get([broker, enable_session_registry]). + emqx:get_config([broker, enable_session_registry]). %% @doc Register a global channel. -spec(register_channel(emqx_types:clientid() diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index d5bf1a61e..056929123 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -43,6 +43,12 @@ , put/2 ]). +-export([ get_raw/1 + , get_raw/2 + , put_raw/1 + , put_raw/2 + ]). + -export([ save_schema_mod_and_names/1 , get_schema_mod/0 , get_schema_mod/1 @@ -61,12 +67,6 @@ , find_listener_conf/3 ]). --export([ get_raw/1 - , get_raw/2 - , put_raw/1 - , put_raw/2 - ]). - -define(CONF, conf). -define(RAW_CONF, raw_conf). -define(PERSIS_SCHEMA_MODS, {?MODULE, schema_mods}). diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index ac66c4daf..dcb50fa4e 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -905,7 +905,7 @@ get_state(Pid) -> tl(tuple_to_list(State)))). get_active_n(Zone, Listener) -> - case emqx_config:get([zones, Zone, listeners, Listener, type]) of + case emqx:get_config([zones, Zone, listeners, Listener, type]) of quic -> 100; _ -> emqx_config:get_listener_conf(Zone, Listener, [tcp, active_n]) end. diff --git a/apps/emqx/src/emqx_flapping.erl b/apps/emqx/src/emqx_flapping.erl index c4a523669..1908430be 100644 --- a/apps/emqx/src/emqx_flapping.erl +++ b/apps/emqx/src/emqx_flapping.erl @@ -160,4 +160,4 @@ start_timer(Zone) -> start_timers() -> lists:foreach(fun({Zone, _ZoneConf}) -> start_timer(Zone) - end, maps:to_list(emqx_config:get([zones], #{}))). + end, maps:to_list(emqx:get_config([zones], #{}))). diff --git a/apps/emqx/src/emqx_global_gc.erl b/apps/emqx/src/emqx_global_gc.erl index 9449efe9a..5192508e5 100644 --- a/apps/emqx/src/emqx_global_gc.erl +++ b/apps/emqx/src/emqx_global_gc.erl @@ -85,7 +85,7 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- ensure_timer(State) -> - case emqx_config:get([node, global_gc_interval]) of + case emqx:get_config([node, global_gc_interval]) of undefined -> State; Interval -> TRef = emqx_misc:start_timer(Interval, run), State#{timer := TRef} diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index f80800768..f39c11305 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -43,7 +43,7 @@ list() -> [{listener_id(ZoneName, LName), LConf} || {ZoneName, LName, LConf} <- do_list()]. do_list() -> - Zones = maps:to_list(emqx_config:get([zones], #{})), + Zones = maps:to_list(emqx:get_config([zones], #{})), lists:append([list(ZoneName, ZoneConf) || {ZoneName, ZoneConf} <- Zones]). list(ZoneName, ZoneConf) -> diff --git a/apps/emqx/src/emqx_os_mon.erl b/apps/emqx/src/emqx_os_mon.erl index 9fd52c21b..85e448f41 100644 --- a/apps/emqx/src/emqx_os_mon.erl +++ b/apps/emqx/src/emqx_os_mon.erl @@ -76,7 +76,7 @@ set_procmem_high_watermark(Float) -> %%-------------------------------------------------------------------- init([]) -> - Opts = emqx_config:get([sysmon, os]), + Opts = emqx:get_config([sysmon, os]), set_mem_check_interval(maps:get(mem_check_interval, Opts)), set_sysmem_high_watermark(maps:get(sysmem_high_watermark, Opts)), set_procmem_high_watermark(maps:get(procmem_high_watermark, Opts)), @@ -91,8 +91,8 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({timeout, _Timer, check}, State) -> - CPUHighWatermark = emqx_config:get([sysmon, os, cpu_high_watermark]) * 100, - CPULowWatermark = emqx_config:get([sysmon, os, cpu_low_watermark]) * 100, + CPUHighWatermark = emqx:get_config([sysmon, os, cpu_high_watermark]) * 100, + CPULowWatermark = emqx:get_config([sysmon, os, cpu_low_watermark]) * 100, _ = case emqx_vm:cpu_util() of %% TODO: should be improved? 0 -> ok; Busy when Busy >= CPUHighWatermark -> @@ -123,7 +123,7 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- start_check_timer() -> - Interval = emqx_config:get([sysmon, os, cpu_check_interval]), + Interval = emqx:get_config([sysmon, os, cpu_check_interval]), case erlang:system_info(system_architecture) of "x86_64-pc-linux-musl" -> ok; _ -> emqx_misc:start_timer(Interval, check) diff --git a/apps/emqx/src/emqx_plugins.erl b/apps/emqx/src/emqx_plugins.erl index 6c99305d4..7bb9c084b 100644 --- a/apps/emqx/src/emqx_plugins.erl +++ b/apps/emqx/src/emqx_plugins.erl @@ -43,7 +43,7 @@ %% @doc Load all plugins when the broker started. -spec(load() -> ok | ignore | {error, term()}). load() -> - ok = load_ext_plugins(emqx_config:get([plugins, expand_plugins_dir], undefined)). + ok = load_ext_plugins(emqx:get_config([plugins, expand_plugins_dir], undefined)). %% @doc Load a Plugin -spec(load(atom()) -> ok | {error, term()}). diff --git a/apps/emqx/src/emqx_router.erl b/apps/emqx/src/emqx_router.erl index 1a5e344f2..8989c3b10 100644 --- a/apps/emqx/src/emqx_router.erl +++ b/apps/emqx/src/emqx_router.erl @@ -250,7 +250,7 @@ delete_trie_route(Route = #route{topic = Topic}) -> %% @private -spec(maybe_trans(function(), list(any())) -> ok | {error, term()}). maybe_trans(Fun, Args) -> - case emqx_config:get([broker, perf, route_lock_type]) of + case emqx:get_config([broker, perf, route_lock_type]) of key -> trans(Fun, Args); global -> diff --git a/apps/emqx/src/emqx_rpc.erl b/apps/emqx/src/emqx_rpc.erl index e950e9e3d..527123745 100644 --- a/apps/emqx/src/emqx_rpc.erl +++ b/apps/emqx/src/emqx_rpc.erl @@ -72,4 +72,4 @@ filter_result(Delivery) -> Delivery. max_client_num() -> - emqx_config:get([rpc, tcp_client_num], ?DefaultClientNum). + emqx:get_config([rpc, tcp_client_num], ?DefaultClientNum). diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 1968c47d8..ccc050165 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -136,11 +136,11 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) -> -spec(strategy() -> strategy()). strategy() -> - emqx_config:get([broker, shared_subscription_strategy]). + emqx:get_config([broker, shared_subscription_strategy]). -spec(ack_enabled() -> boolean()). ack_enabled() -> - emqx_config:get([broker, shared_dispatch_ack_enabled]). + emqx:get_config([broker, shared_dispatch_ack_enabled]). do_dispatch(SubPid, Topic, Msg, _Type) when SubPid =:= self() -> %% Deadlock otherwise diff --git a/apps/emqx/src/emqx_sys.erl b/apps/emqx/src/emqx_sys.erl index 6baae8c1e..70043e2bb 100644 --- a/apps/emqx/src/emqx_sys.erl +++ b/apps/emqx/src/emqx_sys.erl @@ -102,10 +102,10 @@ datetime() -> "~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S])). sys_interval() -> - emqx_config:get([broker, sys_msg_interval]). + emqx:get_config([broker, sys_msg_interval]). sys_heatbeat_interval() -> - emqx_config:get([broker, sys_heartbeat_interval]). + emqx:get_config([broker, sys_heartbeat_interval]). %% @doc Get sys info -spec(info() -> list(tuple())). diff --git a/apps/emqx/src/emqx_sys_mon.erl b/apps/emqx/src/emqx_sys_mon.erl index 0b981ffec..80f5e49ec 100644 --- a/apps/emqx/src/emqx_sys_mon.erl +++ b/apps/emqx/src/emqx_sys_mon.erl @@ -60,7 +60,7 @@ start_timer(State) -> State#{timer := emqx_misc:start_timer(timer:seconds(2), reset)}. sysm_opts() -> - sysm_opts(maps:to_list(emqx_config:get([sysmon, vm])), []). + sysm_opts(maps:to_list(emqx:get_config([sysmon, vm])), []). sysm_opts([], Acc) -> Acc; sysm_opts([{_, disabled}|Opts], Acc) -> diff --git a/apps/emqx/src/emqx_trie.erl b/apps/emqx/src/emqx_trie.erl index 32c176b65..ebfcfcbe3 100644 --- a/apps/emqx/src/emqx_trie.erl +++ b/apps/emqx/src/emqx_trie.erl @@ -270,7 +270,7 @@ match_compact([Word | Words], Prefix, IsWildcard, Acc0) -> lookup_topic(MlTopic). is_compact() -> - emqx_config:get([broker, perf, trie_compaction], true). + emqx:get_config([broker, perf, trie_compaction], true). set_compact(Bool) -> emqx_config:put([broker, perf, trie_compaction], Bool). diff --git a/apps/emqx/src/emqx_vm_mon.erl b/apps/emqx/src/emqx_vm_mon.erl index 13a470959..51710b5b5 100644 --- a/apps/emqx/src/emqx_vm_mon.erl +++ b/apps/emqx/src/emqx_vm_mon.erl @@ -57,8 +57,8 @@ handle_cast(Msg, State) -> {noreply, State}. handle_info({timeout, _Timer, check}, State) -> - ProcHighWatermark = emqx_config:get([sysmon, vm, process_high_watermark]), - ProcLowWatermark = emqx_config:get([sysmon, vm, process_low_watermark]), + ProcHighWatermark = emqx:get_config([sysmon, vm, process_high_watermark]), + ProcLowWatermark = emqx:get_config([sysmon, vm, process_low_watermark]), ProcessCount = erlang:system_info(process_count), case ProcessCount / erlang:system_info(process_limit) of Percent when Percent >= ProcHighWatermark -> @@ -89,5 +89,5 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- start_check_timer() -> - Interval = emqx_config:get([sysmon, vm, process_check_interval]), + Interval = emqx:get_config([sysmon, vm, process_check_interval]), emqx_misc:start_timer(Interval, check). diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index c01420f49..f8b5a7ab6 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -25,7 +25,7 @@ all() -> emqx_ct:all(?MODULE). t_check_pub(_) -> - OldConf = emqx_config:get([zones]), + OldConf = emqx:get_config([zones]), emqx_config:put_zone_conf(default, [mqtt, max_qos_allowed], ?QOS_1), emqx_config:put_zone_conf(default, [mqtt, retain_available], false), timer:sleep(50), @@ -39,7 +39,7 @@ t_check_pub(_) -> emqx_config:put([zones], OldConf). t_check_sub(_) -> - OldConf = emqx_config:get([zones]), + OldConf = emqx:get_config([zones]), SubOpts = #{rh => 0, rap => 0, nl => 0, diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index bd9ec9cfe..7518e5a01 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -36,7 +36,7 @@ stop(_State) -> ok. initialize() -> - AuthNConfig = emqx_config:get([authentication], #{enable => false, + AuthNConfig = emqx:get_config([authentication], #{enable => false, authenticators => []}), initialize(AuthNConfig). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 6197cd685..e3e540de0 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -47,7 +47,7 @@ register_metrics() -> init() -> ok = register_metrics(), emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE), - NRules = [init_rule(Rule) || Rule <- emqx_config:get(?CONF_KEY_PATH, [])], + NRules = [init_rule(Rule) || Rule <- emqx:get_config(?CONF_KEY_PATH, [])], ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NRules]}, -1). lookup() -> diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index 514b0d48b..0452ff96c 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -87,7 +87,7 @@ t_update_rule(_) -> {ok, _} = emqx_authz:update(tail, [?RULE3]), Lists1 = emqx_authz:check_rules([?RULE1, ?RULE2, ?RULE3]), - ?assertMatch(Lists1, emqx_config:get([authorization, rules], [])), + ?assertMatch(Lists1, emqx:get_config([authorization, rules], [])), [#{annotations := #{id := Id1, principal := all, @@ -109,7 +109,7 @@ t_update_rule(_) -> {ok, _} = emqx_authz:update({replace_once, Id3}, ?RULE4), Lists2 = emqx_authz:check_rules([?RULE1, ?RULE2, ?RULE4]), - ?assertMatch(Lists2, emqx_config:get([authorization, rules], [])), + ?assertMatch(Lists2, emqx:get_config([authorization, rules], [])), [#{annotations := #{id := Id1, principal := all, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl index ef4d076a4..4207067fe 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl @@ -39,7 +39,7 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - BridgesConf = emqx_config:get([?APP, bridges], []), + BridgesConf = emqx:get_config([?APP, bridges], []), BridgeSpec = lists:map(fun bridge_spec/1, BridgesConf), SupFlag = #{strategy => one_for_one, intensity => 100, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index adbdbc8e7..fb0e25564 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -98,7 +98,7 @@ stop_listener({Proto, Port, _}) -> listeners() -> [{Protocol, Port, maps:to_list(maps:without([protocol, port], Map))} || Map = #{protocol := Protocol,port := Port} - <- emqx_config:get([emqx_dashboard, listeners], [])]. + <- emqx:get_config([emqx_dashboard, listeners], [])]. listener_name(Proto) -> list_to_atom(atom_to_list(Proto) ++ ":dashboard"). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index b32d3d346..982756805 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -201,7 +201,7 @@ add_default_user() -> add_default_user(binenv(default_username), binenv(default_password)). binenv(Key) -> - iolist_to_binary(emqx_config:get([emqx_dashboard, Key], "")). + iolist_to_binary(emqx:get_config([emqx_dashboard, Key], "")). add_default_user(Username, Password) when ?EMPTY_KEY(Username) orelse ?EMPTY_KEY(Password) -> igonre; diff --git a/apps/emqx_dashboard/src/emqx_dashboard_collection.erl b/apps/emqx_dashboard/src/emqx_dashboard_collection.erl index 91d60e1ab..8b0576342 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_collection.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_collection.erl @@ -58,7 +58,7 @@ get_collect() -> gen_server:call(whereis(?MODULE), get_collect). init([]) -> timer(next_interval(), collect), timer(get_today_remaining_seconds(), clear_expire_data), - ExpireInterval = emqx_config:get([emqx_dashboard, monitor, interval], ?EXPIRE_INTERVAL), + ExpireInterval = emqx:get_config([emqx_dashboard, monitor, interval], ?EXPIRE_INTERVAL), State = #{ count => count(), expire_interval => ExpireInterval, @@ -78,7 +78,7 @@ next_interval() -> (1000 * interval()) - (erlang:system_time(millisecond) rem (1000 * interval())) - 1. interval() -> - emqx_config:get([?APP, sample_interval], ?DEFAULT_INTERVAL). + emqx:get_config([?APP, sample_interval], ?DEFAULT_INTERVAL). count() -> 60 div interval(). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index fdba7fb7e..432a64621 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -148,7 +148,7 @@ jwk(Username, Password, Salt) -> }. jwt_expiration_time() -> - ExpTime = emqx_config:get([emqx_dashboard, token_expired_time], ?EXPTIME), + ExpTime = emqx:get_config([emqx_dashboard, token_expired_time], ?EXPTIME), erlang:system_time(millisecond) + ExpTime. salt() -> diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index 17527ca3a..52cea80fb 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -27,7 +27,7 @@ ]). load_bridges() -> - Bridges = emqx_config:get([emqx_data_bridge, bridges], []), + Bridges = emqx:get_config([emqx_data_bridge, bridges], []), emqx_data_bridge_monitor:ensure_all_started(Bridges). resource_type(mysql) -> emqx_connector_mysql; diff --git a/apps/emqx_exhook/src/emqx_exhook_sup.erl b/apps/emqx_exhook/src/emqx_exhook_sup.erl index 32f8fa472..60a6a2915 100644 --- a/apps/emqx_exhook/src/emqx_exhook_sup.erl +++ b/apps/emqx_exhook/src/emqx_exhook_sup.erl @@ -58,7 +58,7 @@ request_options() -> }. env(Key, Def) -> - emqx_config:get([exhook, Key], Def). + emqx:get_config([exhook, Key], Def). %%-------------------------------------------------------------------- %% APIs diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index b27319eac..adc546767 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -79,4 +79,4 @@ load_gateway_by_default([{Type, Confs}|More]) -> load_gateway_by_default(More). confs() -> - maps:to_list(emqx_config:get([gateway], [])). + maps:to_list(emqx:get_config([gateway], [])). diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 7fb700c55..4a7fefb2d 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -590,7 +590,7 @@ check_row_limit([Tab|Tables], Limit) -> end. max_row_limit() -> - emqx_config:get([?APP, max_row_limit], ?MAX_ROW_LIMIT). + emqx:get_config([?APP, max_row_limit], ?MAX_ROW_LIMIT). table_size(Tab) -> ets:info(Tab, size). diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 15849c294..b54e357d6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -56,7 +56,7 @@ api_spec() -> config_apis() -> [config_api(ConfPath, Schema) || {ConfPath, Schema} <- - get_conf_schema(emqx_config:get([]), ?MAX_DEPTH), is_core_conf(ConfPath)]. + get_conf_schema(emqx:get_config([]), ?MAX_DEPTH), is_core_conf(ConfPath)]. config_api(ConfPath, Schema) -> Path = path_join(ConfPath), @@ -131,7 +131,7 @@ config_reset(post, Req) -> get_full_config() -> emqx_map_lib:jsonable_map( - emqx_config:fill_defaults(emqx_config:get_raw([]))). + emqx_config:fill_defaults(emqx:get_raw_config([]))). conf_path_from_querystr(Req) -> case proplists:get_value(<<"conf_path">>, cowboy_req:parse_qs(Req)) of diff --git a/apps/emqx_management/src/emqx_mgmt_auth.erl b/apps/emqx_management/src/emqx_mgmt_auth.erl index 7fb120017..73ec37fc2 100644 --- a/apps/emqx_management/src/emqx_mgmt_auth.erl +++ b/apps/emqx_management/src/emqx_mgmt_auth.erl @@ -68,7 +68,7 @@ mnesia(copy) -> %%-------------------------------------------------------------------- -spec(add_default_app() -> list()). add_default_app() -> - Apps = emqx_config:get([?APP, applications], []), + Apps = emqx:get_config([?APP, applications], []), [ begin case {AppId, AppSecret} of {undefined, _} -> ok; diff --git a/apps/emqx_management/src/emqx_mgmt_http.erl b/apps/emqx_management/src/emqx_mgmt_http.erl index c795e1de7..7bd393904 100644 --- a/apps/emqx_management/src/emqx_mgmt_http.erl +++ b/apps/emqx_management/src/emqx_mgmt_http.erl @@ -94,7 +94,7 @@ stop_listener({Proto, Port, _}) -> listeners() -> [{Protocol, Port, maps:to_list(maps:without([protocol, port], Map))} || Map = #{protocol := Protocol,port := Port} - <- emqx_config:get([emqx_management, listeners], [])]. + <- emqx:get_config([emqx_management, listeners], [])]. listener_name(Proto) -> list_to_atom(atom_to_list(Proto) ++ ":management"). diff --git a/apps/emqx_modules/src/emqx_delayed.erl b/apps/emqx_modules/src/emqx_delayed.erl index 5e1754f4b..00532e25e 100644 --- a/apps/emqx_modules/src/emqx_delayed.erl +++ b/apps/emqx_modules/src/emqx_delayed.erl @@ -104,7 +104,7 @@ on_message_publish(Msg) -> -spec(start_link() -> emqx_types:startlink_ret()). start_link() -> - Opts = emqx_config:get([delayed], #{}), + Opts = emqx:get_config([delayed], #{}), gen_server:start_link({local, ?SERVER}, ?MODULE, [Opts], []). -spec(store(#delayed_message{}) -> ok | {error, atom()}). diff --git a/apps/emqx_modules/src/emqx_delayed_api.erl b/apps/emqx_modules/src/emqx_delayed_api.erl index 06de3ab13..9d4edd940 100644 --- a/apps/emqx_modules/src/emqx_delayed_api.erl +++ b/apps/emqx_modules/src/emqx_delayed_api.erl @@ -179,4 +179,4 @@ rpc_call(Node, Module, Fun, Args) -> end. get_status() -> - emqx_config:get([delayed, enable], true). + emqx:get_config([delayed, enable], true). diff --git a/apps/emqx_modules/src/emqx_event_message.erl b/apps/emqx_modules/src/emqx_event_message.erl index 3bdc54c2b..5017ed3e4 100644 --- a/apps/emqx_modules/src/emqx_event_message.erl +++ b/apps/emqx_modules/src/emqx_event_message.erl @@ -38,7 +38,7 @@ -endif. enable() -> - Topics = emqx_config:get([event_message, topics], []), + Topics = emqx:get_config([event_message, topics], []), lists:foreach(fun(Topic) -> case Topic of <<"$event/client_connected">> -> @@ -61,7 +61,7 @@ enable() -> end, Topics). disable() -> - Topics = emqx_config:get([event_message, topics], []), + Topics = emqx:get_config([event_message, topics], []), lists:foreach(fun(Topic) -> case Topic of <<"$event/client_connected">> -> diff --git a/apps/emqx_modules/src/emqx_modules_app.erl b/apps/emqx_modules/src/emqx_modules_app.erl index e969fc6dd..889c566b1 100644 --- a/apps/emqx_modules/src/emqx_modules_app.erl +++ b/apps/emqx_modules/src/emqx_modules_app.erl @@ -32,17 +32,17 @@ stop(_State) -> ok. maybe_enable_modules() -> - emqx_config:get([delayed, enable], true) andalso emqx_delayed:enable(), - emqx_config:get([telemetry, enable], true) andalso emqx_telemetry:enable(), - emqx_config:get([recon, enable], true) andalso emqx_recon:enable(), + emqx:get_config([delayed, enable], true) andalso emqx_delayed:enable(), + emqx:get_config([telemetry, enable], true) andalso emqx_telemetry:enable(), + emqx:get_config([recon, enable], true) andalso emqx_recon:enable(), emqx_event_message:enable(), emqx_rewrite:enable(), emqx_topic_metrics:enable(). maybe_disable_modules() -> - emqx_config:get([delayed, enable], true) andalso emqx_delayed:disable(), - emqx_config:get([telemetry, enable], true) andalso emqx_telemetry:disable(), - emqx_config:get([recon, enable], true) andalso emqx_recon:disable(), + emqx:get_config([delayed, enable], true) andalso emqx_delayed:disable(), + emqx:get_config([telemetry, enable], true) andalso emqx_telemetry:disable(), + emqx:get_config([recon, enable], true) andalso emqx_recon:disable(), emqx_event_message:disable(), emqx_rewrite:disable(), emqx_topic_metrics:disable(). diff --git a/apps/emqx_modules/src/emqx_rewrite.erl b/apps/emqx_modules/src/emqx_rewrite.erl index ec5a7d5ba..9a6c0574b 100644 --- a/apps/emqx_modules/src/emqx_rewrite.erl +++ b/apps/emqx_modules/src/emqx_rewrite.erl @@ -43,7 +43,7 @@ %%-------------------------------------------------------------------- enable() -> - Rules = emqx_config:get([rewrite, rules], []), + Rules = emqx:get_config([rewrite, rules], []), register_hook(Rules). disable() -> @@ -52,10 +52,10 @@ disable() -> emqx_hooks:del('message.publish', {?MODULE, rewrite_publish}). list() -> - maps:get(<<"rules">>, emqx_config:get_raw([<<"rewrite">>], #{}), []). + maps:get(<<"rules">>, emqx:get_raw_config([<<"rewrite">>], #{}), []). update(Rules0) -> - Rewrite = emqx_config:get_raw([<<"rewrite">>], #{}), + Rewrite = emqx:get_raw_config([<<"rewrite">>], #{}), {ok, #{config := Config}} = emqx:update_config([rewrite], maps:put(<<"rules">>, Rules0, Rewrite)), Rules = maps:get(rules, maps:get(rewrite, Config, #{}), []), diff --git a/apps/emqx_modules/src/emqx_telemetry.erl b/apps/emqx_modules/src/emqx_telemetry.erl index c9a78e736..aa207ac95 100644 --- a/apps/emqx_modules/src/emqx_telemetry.erl +++ b/apps/emqx_modules/src/emqx_telemetry.erl @@ -107,7 +107,7 @@ mnesia(copy) -> %%-------------------------------------------------------------------- start_link() -> - Opts = emqx_config:get([telemetry], #{}), + Opts = emqx:get_config([telemetry], #{}), gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). stop() -> @@ -120,7 +120,7 @@ disable() -> gen_server:call(?MODULE, disable). get_status() -> - emqx_config:get([telemetry, enable], true). + emqx:get_config([telemetry, enable], true). get_uuid() -> gen_server:call(?MODULE, get_uuid). diff --git a/apps/emqx_modules/src/emqx_topic_metrics.erl b/apps/emqx_modules/src/emqx_topic_metrics.erl index 5e6a3bc98..fa226c9a0 100644 --- a/apps/emqx_modules/src/emqx_topic_metrics.erl +++ b/apps/emqx_modules/src/emqx_topic_metrics.erl @@ -137,7 +137,7 @@ on_message_dropped(#message{topic = Topic}, _, _) -> end. start_link() -> - Opts = emqx_config:get([topic_metrics], #{}), + Opts = emqx:get_config([topic_metrics], #{}), gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). stop() -> diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl index 3fac97e86..e51a4b6a6 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl @@ -45,7 +45,7 @@ -spec save_files_return_opts(opts_input(), atom() | string() | binary(), string() | binary()) -> opts(). save_files_return_opts(Options, SubDir, ResId) -> - Dir = filename:join([emqx_config:get([node, data_dir]), SubDir, ResId]), + Dir = filename:join([emqx:get_config([node, data_dir]), SubDir, ResId]), save_files_return_opts(Options, Dir). %% @doc Parse ssl options input. @@ -76,7 +76,7 @@ save_files_return_opts(Options, Dir) -> %% empty string is returned if the input is empty. -spec save_file(file_input(), atom() | string() | binary()) -> string(). save_file(Param, SubDir) -> - Dir = filename:join([emqx_config:get([node, data_dir]), SubDir]), + Dir = filename:join([emqx:get_config([node, data_dir]), SubDir]), do_save_file(Param, Dir). filter([]) -> []; diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index 63b8b6a10..a2ac3b708 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -106,7 +106,7 @@ prometheus_api() -> % {"/prometheus/stats", Metadata, stats}. prometheus(get, _Request) -> - Response = emqx_config:get_raw([<<"prometheus">>], #{}), + Response = emqx:get_raw_config([<<"prometheus">>], #{}), {200, Response}; prometheus(put, Request) -> @@ -128,11 +128,11 @@ prometheus(put, Request) -> enable_prometheus(true) -> ok = emqx_prometheus_sup:stop_child(?APP), - emqx_prometheus_sup:start_child(?APP, emqx_config:get([prometheus], #{})), + emqx_prometheus_sup:start_child(?APP, emqx:get_config([prometheus], #{})), {200}; enable_prometheus(false) -> _ = emqx_prometheus_sup:stop_child(?APP), {200}. get_raw(Key, Def) -> - emqx_config:get_raw([<<"prometheus">>] ++ [Key], Def). + emqx:get_raw_config([<<"prometheus">>] ++ [Key], Def). diff --git a/apps/emqx_prometheus/src/emqx_prometheus_app.erl b/apps/emqx_prometheus/src/emqx_prometheus_app.erl index f4d4fd164..4f954a792 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_app.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_app.erl @@ -34,9 +34,9 @@ stop(_State) -> ok. maybe_enable_prometheus() -> - case emqx_config:get([prometheus, enable], false) of + case emqx:get_config([prometheus, enable], false) of true -> - emqx_prometheus_sup:start_child(?APP, emqx_config:get([prometheus], #{})); + emqx_prometheus_sup:start_child(?APP, emqx:get_config([prometheus], #{})); false -> ok end. diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index acd119dee..dfbe5cc69 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -129,7 +129,7 @@ deliver(Result, #{context_id := Id} = Context, Pid, Topic, Cursor) -> false -> ok; _ -> - #{msg_deliver_quota := MaxDeliverNum} = emqx_config:get([?APP, flow_control]), + #{msg_deliver_quota := MaxDeliverNum} = emqx:get_config([?APP, flow_control]), case MaxDeliverNum of 0 -> _ = [Pid ! {deliver, Topic, Msg} || Msg <- Result], @@ -150,7 +150,7 @@ get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' : timestamp = Ts}) -> Ts + Interval * 1000; get_expiry_time(#message{timestamp = Ts}) -> - Interval = emqx_config:get([?APP, msg_expiry_interval], ?DEF_EXPIRY_INTERVAL), + Interval = emqx:get_config([?APP, msg_expiry_interval], ?DEF_EXPIRY_INTERVAL), case Interval of 0 -> 0; _ -> Ts + Interval @@ -173,7 +173,7 @@ delete(Topic) -> init([]) -> init_shared_context(), State = new_state(), - #{enable := Enable} = Cfg = emqx_config:get([?APP]), + #{enable := Enable} = Cfg = emqx:get_config([?APP]), {ok, case Enable of true -> @@ -209,7 +209,7 @@ handle_cast(Msg, State) -> handle_info(clear_expired, #{context := Context} = State) -> Mod = get_backend_module(), Mod:clear_expired(Context), - Interval = emqx_config:get([?APP, msg_clear_interval], ?DEF_EXPIRY_INTERVAL), + Interval = emqx:get_config([?APP, msg_clear_interval], ?DEF_EXPIRY_INTERVAL), {noreply, State#{clear_timer := add_timer(Interval, clear_expired)}, hibernate}; handle_info(release_deliver_quota, #{context := Context, wait_quotas := Waits} = State) -> @@ -225,7 +225,7 @@ handle_info(release_deliver_quota, #{context := Context, wait_quotas := Waits} = end, Waits2) end, - Interval = emqx_config:get([?APP, flow_control, quota_release_interval]), + Interval = emqx:get_config([?APP, flow_control, quota_release_interval]), {noreply, State#{release_quota_timer := add_timer(Interval, release_deliver_quota), wait_quotas := []}}; @@ -258,7 +258,7 @@ new_context(Id) -> #{context_id => Id}. is_too_big(Size) -> - Limit = emqx_config:get([?APP, max_payload_size], ?DEF_MAX_PAYLOAD_SIZE), + Limit = emqx:get_config([?APP, max_payload_size], ?DEF_MAX_PAYLOAD_SIZE), Limit > 0 andalso (Size > Limit). %% @private @@ -332,7 +332,7 @@ insert_shared_context(Key, Term) -> -spec get_msg_deliver_quota() -> non_neg_integer(). get_msg_deliver_quota() -> - emqx_config:get([?APP, flow_control, msg_deliver_quota]). + emqx:get_config([?APP, flow_control, msg_deliver_quota]). -spec update_config(state(), hocons:config()) -> state(). update_config(#{clear_timer := ClearTimer, @@ -342,7 +342,7 @@ update_config(#{clear_timer := ClearTimer, flow_control := #{quota_release_interval := QuotaInterval}, msg_clear_interval := ClearInterval} = Conf, - #{config := OldConfig} = emqx_config:get([?APP]), + #{config := OldConfig} = emqx:get_config([?APP]), case Enable of true -> @@ -416,7 +416,7 @@ check_timer(Timer, _, _) -> -spec get_backend_module() -> backend(). get_backend_module() -> - #{type := Backend} = emqx_config:get([?APP, config]), + #{type := Backend} = emqx:get_config([?APP, config]), ModName = if Backend =:= built_in_database -> mnesia; true -> diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 1b5b8adcc..a0b72c858 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -35,7 +35,7 @@ ]). lookup_config(_Bindings, _Params) -> - Config = emqx_config:get([emqx_retainer]), + Config = emqx:get_config([emqx_retainer]), return({ok, Config}). update_config(_Bindings, Params) -> diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl index 34e3e49db..1c9956050 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl @@ -130,7 +130,7 @@ read_message(_, Topic) -> {ok, read_messages(Topic)}. match_messages(_, Topic, Cursor) -> - MaxReadNum = emqx_config:get([?APP, flow_control, max_read_number]), + MaxReadNum = emqx:get_config([?APP, flow_control, max_read_number]), case Cursor of undefined -> case MaxReadNum of @@ -227,7 +227,7 @@ make_match_spec(Filter) -> -spec is_table_full() -> boolean(). is_table_full() -> - #{max_retained_messages := Limit} = emqx_config:get([?APP, config]), + #{max_retained_messages := Limit} = emqx:get_config([?APP, config]), Limit > 0 andalso (table_size() >= Limit). -spec table_size() -> non_neg_integer(). diff --git a/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl index 8d17ee6f5..194fc4096 100644 --- a/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_rule_actions/src/emqx_bridge_mqtt_actions.erl @@ -506,7 +506,7 @@ connect(Options) when is_list(Options) -> connect(Options = #{disk_cache := DiskCache, ecpool_worker_id := Id, pool_name := Pool}) -> Options0 = case DiskCache of true -> - DataDir = filename:join([emqx_config:get([node, data_dir]), replayq, Pool, integer_to_list(Id)]), + DataDir = filename:join([emqx:get_config([node, data_dir]), replayq, Pool, integer_to_list(Id)]), QueueOption = #{replayq_dir => DataDir}, Options#{queue => QueueOption}; false -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 6eacced42..8154170b0 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -595,4 +595,4 @@ printable_maps(Headers) -> ignore_sys_message(#message{flags = Flags}) -> maps:get(sys, Flags, false) andalso - emqx_config:get([emqx_rule_engine, ignore_sys_message]). + emqx:get_config([emqx_rule_engine, ignore_sys_message]). diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index 25f0d1879..3325d8c71 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -84,7 +84,7 @@ statsd_api() -> [{"/statsd", Metadata, statsd}]. statsd(get, _Request) -> - Response = emqx_config:get_raw([<<"statsd">>], #{}), + Response = emqx:get_raw_config([<<"statsd">>], #{}), {200, Response}; statsd(put, Request) -> @@ -96,11 +96,11 @@ statsd(put, Request) -> enable_statsd(true) -> ok = emqx_statsd_sup:stop_child(?APP), - emqx_statsd_sup:start_child(?APP, emqx_config:get([statsd], #{})), + emqx_statsd_sup:start_child(?APP, emqx:get_config([statsd], #{})), {200}; enable_statsd(false) -> _ = emqx_statsd_sup:stop_child(?APP), {200}. get_raw(Key, Def) -> - emqx_config:get_raw([<<"statsd">>]++ [Key], Def). + emqx:get_raw_config([<<"statsd">>]++ [Key], Def). diff --git a/apps/emqx_statsd/src/emqx_statsd_app.erl b/apps/emqx_statsd/src/emqx_statsd_app.erl index 6dd9dc5c7..4a5ff0496 100644 --- a/apps/emqx_statsd/src/emqx_statsd_app.erl +++ b/apps/emqx_statsd/src/emqx_statsd_app.erl @@ -32,9 +32,9 @@ stop(_) -> ok. maybe_enable_statsd() -> - case emqx_config:get([statsd, enable], false) of + case emqx:get_config([statsd, enable], false) of true -> - emqx_statsd_sup:start_child(?APP, emqx_config:get([statsd], #{})); + emqx_statsd_sup:start_child(?APP, emqx:get_config([statsd], #{})); false -> ok end. From f01b77e4fe050d0e72a576674a6372d0567d0aab Mon Sep 17 00:00:00 2001 From: turtleDeng Date: Wed, 18 Aug 2021 16:26:15 +0800 Subject: [PATCH 03/20] refactor(event-message): refactor event_message * refactor(event-message): refactor event_message configuration * feat(event-message): add event_message REST API --- apps/emqx_modules/etc/emqx_modules.conf | 19 ++-- apps/emqx_modules/include/emqx_modules.hrl | 8 -- apps/emqx_modules/src/emqx_event_message.erl | 80 ++++++++------- .../src/emqx_event_message_api.erl | 99 +++++++++++++++++++ apps/emqx_modules/src/emqx_modules_schema.erl | 25 ++--- .../test/emqx_event_message_SUITE.erl | 34 +++---- 6 files changed, 176 insertions(+), 89 deletions(-) create mode 100644 apps/emqx_modules/src/emqx_event_message_api.erl diff --git a/apps/emqx_modules/etc/emqx_modules.conf b/apps/emqx_modules/etc/emqx_modules.conf index 92f563342..bae8fd3ed 100644 --- a/apps/emqx_modules/etc/emqx_modules.conf +++ b/apps/emqx_modules/etc/emqx_modules.conf @@ -11,17 +11,14 @@ telemetry: { enable: true } - -event_message: { - topics: [ - "$event/client_connected", - "$event/client_disconnected", - "$event/session_subscribed", - "$event/session_unsubscribed", - "$event/message_delivered", - "$event/message_acked", - "$event/message_dropped" - ] +event_message { + "$event/client_connected": true + "$event/client_disconnected": true + # "$event/client_subscribed": false + # "$event/client_unsubscribed": false + # "$event/message_delivered": false + # "$event/message_acked": false + # "$event/message_dropped": false } topic_metrics:{ diff --git a/apps/emqx_modules/include/emqx_modules.hrl b/apps/emqx_modules/include/emqx_modules.hrl index b7cdb154e..334173015 100644 --- a/apps/emqx_modules/include/emqx_modules.hrl +++ b/apps/emqx_modules/include/emqx_modules.hrl @@ -3,11 +3,3 @@ %% Interval for reporting telemetry data, Default: 7d -define(REPORT_INTERVAR, 604800). - --define(BASE_TOPICS, [<<"$event/client_connected">>, - <<"$event/client_disconnected">>, - <<"$event/session_subscribed">>, - <<"$event/session_unsubscribed">>, - <<"$event/message_delivered">>, - <<"$event/message_acked">>, - <<"$event/message_dropped">>]). diff --git a/apps/emqx_modules/src/emqx_event_message.erl b/apps/emqx_modules/src/emqx_event_message.erl index 5017ed3e4..842ddae04 100644 --- a/apps/emqx_modules/src/emqx_event_message.erl +++ b/apps/emqx_modules/src/emqx_event_message.erl @@ -20,14 +20,16 @@ -include_lib("emqx/include/logger.hrl"). -include("emqx_modules.hrl"). --export([ enable/0 +-export([ list/0 + , update/1 + , enable/0 , disable/0 ]). -export([ on_client_connected/2 , on_client_disconnected/3 - , on_session_subscribed/3 - , on_session_unsubscribed/3 + , on_client_subscribed/3 + , on_client_unsubscribed/3 , on_message_dropped/3 , on_message_delivered/2 , on_message_acked/2 @@ -37,51 +39,59 @@ -export([reason/1]). -endif. +list() -> + emqx:get_config([event_message], #{}). + +update(Params) -> + disable(), + {ok, _} = emqx:update_config([event_message], Params), + enable(). + enable() -> - Topics = emqx:get_config([event_message, topics], []), - lists:foreach(fun(Topic) -> + lists:foreach(fun({_Topic, false}) -> ok; + ({Topic, true}) -> case Topic of - <<"$event/client_connected">> -> + '$event/client_connected' -> emqx_hooks:put('client.connected', {?MODULE, on_client_connected, []}); - <<"$event/client_disconnected">> -> + '$event/client_disconnected' -> emqx_hooks:put('client.disconnected', {?MODULE, on_client_disconnected, []}); - <<"$event/session_subscribed">> -> - emqx_hooks:put('session.subscribed', {?MODULE, on_session_subscribed, []}); - <<"$event/session_unsubscribed">> -> - emqx_hooks:put('session.unsubscribed', {?MODULE, on_session_unsubscribed, []}); - <<"$event/message_delivered">> -> + '$event/client_subscribed' -> + emqx_hooks:put('session.subscribed', {?MODULE, on_client_subscribed, []}); + '$event/client_unsubscribed' -> + emqx_hooks:put('session.unsubscribed', {?MODULE, on_client_unsubscribed, []}); + '$event/message_delivered' -> emqx_hooks:put('message.delivered', {?MODULE, on_message_delivered, []}); - <<"$event/message_acked">> -> + '$event/message_acked' -> emqx_hooks:put('message.acked', {?MODULE, on_message_acked, []}); - <<"$event/message_dropped">> -> + '$event/message_dropped' -> emqx_hooks:put('message.dropped', {?MODULE, on_message_dropped, []}); _ -> ok end - end, Topics). + end, maps:to_list(list())). disable() -> - Topics = emqx:get_config([event_message, topics], []), - lists:foreach(fun(Topic) -> + lists:foreach(fun({_Topic, false}) -> ok; + ({Topic, true}) -> case Topic of - <<"$event/client_connected">> -> + '$event/client_connected' -> emqx_hooks:del('client.connected', {?MODULE, on_client_connected}); - <<"$event/client_disconnected">> -> + '$event/client_disconnected' -> emqx_hooks:del('client.disconnected', {?MODULE, on_client_disconnected}); - <<"$event/session_subscribed">> -> - emqx_hooks:del('session.subscribed', {?MODULE, on_session_subscribed}); - <<"$event/session_unsubscribed">> -> - emqx_hooks:del('session.unsubscribed', {?MODULE, on_session_unsubscribed}); - <<"$event/message_delivered">> -> + '$event/client_subscribed' -> + emqx_hooks:del('session.subscribed', {?MODULE, on_client_subscribed}); + '$event/client_unsubscribed' -> + emqx_hooks:del('session.unsubscribed', {?MODULE, on_client_unsubscribed}); + '$event/message_delivered' -> emqx_hooks:del('message.delivered', {?MODULE, on_message_delivered}); - <<"$event/message_acked">> -> + '$event/message_acked' -> emqx_hooks:del('message.acked', {?MODULE, on_message_acked}); - <<"$event/message_dropped">> -> + '$event/message_dropped' -> emqx_hooks:del('message.dropped', {?MODULE, on_message_dropped}); _ -> ok end - end, ?BASE_TOPICS -- Topics). + end, maps:to_list(list())). %%-------------------------------------------------------------------- %% Callbacks @@ -90,7 +100,6 @@ disable() -> on_client_connected(ClientInfo, ConnInfo) -> Payload0 = common_infos(ClientInfo, ConnInfo), Payload = Payload0#{ - connack => 0, %% XXX: connack will be removed in 5.0 keepalive => maps:get(keepalive, ConnInfo, 0), clean_start => maps:get(clean_start, ConnInfo, true), expiry_interval => maps:get(expiry_interval, ConnInfo, 0), @@ -108,8 +117,8 @@ on_client_disconnected(ClientInfo, }, publish_event_msg(<<"$event/client_disconnected">>, Payload). -on_session_subscribed(_ClientInfo = #{clientid := ClientId, - username := Username}, +on_client_subscribed(_ClientInfo = #{clientid := ClientId, + username := Username}, Topic, SubOpts) -> Payload = #{clientid => ClientId, username => Username, @@ -117,17 +126,17 @@ on_session_subscribed(_ClientInfo = #{clientid := ClientId, subopts => SubOpts, ts => erlang:system_time(millisecond) }, - publish_event_msg(<<"$event/session_subscribed">>, Payload). + publish_event_msg(<<"$event/client_subscribed">>, Payload). -on_session_unsubscribed(_ClientInfo = #{clientid := ClientId, - username := Username}, +on_client_unsubscribed(_ClientInfo = #{clientid := ClientId, + username := Username}, Topic, _SubOpts) -> Payload = #{clientid => ClientId, username => Username, topic => Topic, ts => erlang:system_time(millisecond) }, - publish_event_msg(<<"$event/session_unsubscribed">>, Payload). + publish_event_msg(<<"$event/client_unsubscribed">>, Payload). on_message_dropped(Message = #message{from = ClientId}, _, Reason) -> case ignore_sys_message(Message) of @@ -201,8 +210,7 @@ common_infos( ipaddress => ntoa(PeerHost), sockport => SockPort, proto_name => ProtoName, - proto_ver => ProtoVer, - ts => erlang:system_time(millisecond) + proto_ver => ProtoVer }. make_msg(Topic, Payload) -> diff --git a/apps/emqx_modules/src/emqx_event_message_api.erl b/apps/emqx_modules/src/emqx_event_message_api.erl new file mode 100644 index 000000000..86c3255e1 --- /dev/null +++ b/apps/emqx_modules/src/emqx_event_message_api.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_event_message_api). + +-behaviour(minirest_api). + +-export([api_spec/0]). + +-export([event_message/2]). + + +api_spec() -> + {[event_message_api()], [event_message_schema()]}. + +event_message_schema() -> + #{ + type => object, + properties => #{ + '$event/client_connected' => #{ + type => boolean, + description => <<"Client connected event">>, + example => get_raw(<<"$event/client_connected">>) + }, + '$event/client_disconnected' => #{ + type => boolean, + description => <<"client_disconnected">>, + example => get_raw(<<"Client disconnected event">>) + }, + '$event/client_subscribed' => #{ + type => boolean, + description => <<"client_subscribed">>, + example => get_raw(<<"Client subscribed event">>) + }, + '$event/client_unsubscribed' => #{ + type => boolean, + description => <<"client_unsubscribed">>, + example => get_raw(<<"Client unsubscribed event">>) + }, + '$event/message_delivered' => #{ + type => boolean, + description => <<"message_delivered">>, + example => get_raw(<<"Message delivered event">>) + }, + '$event/message_acked' => #{ + type => boolean, + description => <<"message_acked">>, + example => get_raw(<<"Message acked event">>) + }, + '$event/message_dropped' => #{ + type => boolean, + description => <<"message_dropped">>, + example => get_raw(<<"Message dropped event">>) + } + } + }. + +event_message_api() -> + Path = "/mqtt/event_message", + Metadata = #{ + get => #{ + description => <<"Event Message">>, + responses => #{ + <<"200">> => + emqx_mgmt_util:response_schema(<<>>, event_message_schema())}}, + post => #{ + description => <<"">>, + 'requestBody' => emqx_mgmt_util:request_body_schema(event_message_schema()), + responses => #{ + <<"200">> => + emqx_mgmt_util:response_schema(<<>>, event_message_schema()) + } + } + }, + {Path, Metadata, event_message}. + +event_message(get, _Request) -> + {200, emqx_event_message:list()}; + +event_message(post, Request) -> + {ok, Body, _} = cowboy_req:read_body(Request), + Params = emqx_json:decode(Body, [return_maps]), + _ = emqx_event_message:update(Params), + {200, emqx_event_message:list()}. + +get_raw(Key) -> + emqx_config:get_raw([<<"event_message">>] ++ [Key], false). diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index 0097fdbbe..0c5e716bf 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -45,8 +45,15 @@ fields("rewrite") -> [ {rules, hoconsc:array(hoconsc:ref(?MODULE, "rules"))} ]; + fields("event_message") -> - [ {topics, fun topics/1} + [ {"$event/client_connected", emqx_schema:t(boolean(), undefined, false)} + , {"$event/client_disconnected", emqx_schema:t(boolean(), undefined, false)} + , {"$event/client_subscribed", emqx_schema:t(boolean(), undefined, false)} + , {"$event/client_unsubscribed", emqx_schema:t(boolean(), undefined, false)} + , {"$event/message_delivered", emqx_schema:t(boolean(), undefined, false)} + , {"$event/message_acked", emqx_schema:t(boolean(), undefined, false)} + , {"$event/message_dropped", emqx_schema:t(boolean(), undefined, false)} ]; fields("topic_metrics") -> @@ -60,19 +67,3 @@ fields("rules") -> , {dest_topic, emqx_schema:t(binary())} ]. -topics(type) -> hoconsc:array(binary()); -topics(default) -> []; -% topics(validator) -> [ -% fun(Conf) -> -% case lists:member(Conf, ["$event/client_connected", -% "$event/client_disconnected", -% "$event/session_subscribed", -% "$event/session_unsubscribed", -% "$event/message_delivered", -% "$event/message_acked", -% "$event/message_dropped"]) of -% true -> ok; -% false -> {error, "Bad event topic"} -% end -% end]; -topics(_) -> undefined. diff --git a/apps/emqx_modules/test/emqx_event_message_SUITE.erl b/apps/emqx_modules/test/emqx_event_message_SUITE.erl index 97235ac1f..526113bcc 100644 --- a/apps/emqx_modules/test/emqx_event_message_SUITE.erl +++ b/apps/emqx_modules/test/emqx_event_message_SUITE.erl @@ -24,16 +24,14 @@ -define(EVENT_MESSAGE, <<""" event_message: { - topics : [ - \"$event/client_connected\", - \"$event/client_disconnected\", - \"$event/session_subscribed\", - \"$event/session_unsubscribed\", - \"$event/message_delivered\", - \"$event/message_acked\", - \"$event/message_dropped\" - ]}""">>). - + \"$event/client_connected\": true + \"$event/client_disconnected\": true + \"$event/client_subscribed\": true + \"$event/client_unsubscribed\": true + \"$event/message_delivered\": true + \"$event/message_acked\": true + \"$event/message_dropped\": true + }""">>). all() -> emqx_ct:all(?MODULE). @@ -56,12 +54,12 @@ t_event_topic(_) -> {ok, _} = emqtt:connect(C2), ok = recv_connected(<<"clientid">>), - {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/session_subscribed">>, qos1), + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/client_subscribed">>, qos1), _ = receive_publish(100), timer:sleep(50), {ok, _, [?QOS_1]} = emqtt:subscribe(C2, <<"test_sub">>, qos1), ok = recv_subscribed(<<"clientid">>), - emqtt:unsubscribe(C1, <<"$event/session_subscribed">>), + emqtt:unsubscribe(C1, <<"$event/client_subscribed">>), timer:sleep(50), {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/message_delivered">>, qos1), @@ -77,7 +75,7 @@ t_event_topic(_) -> ok= emqtt:publish(C2, <<"test_sub1">>, <<"test">>), recv_message_dropped(<<"clientid">>), - {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/session_unsubscribed">>, qos1), + {ok, _, [?QOS_1]} = emqtt:subscribe(C1, <<"$event/client_unsubscribed">>, qos1), _ = emqtt:unsubscribe(C2, <<"test_sub">>), ok = recv_unsubscribed(<<"clientid">>), @@ -101,12 +99,14 @@ recv_connected(ClientId) -> <<"ipaddress">> := <<"127.0.0.1">>, <<"proto_name">> := <<"MQTT">>, <<"proto_ver">> := ?MQTT_PROTO_V4, - <<"connack">> := ?RC_SUCCESS, - <<"clean_start">> := true}, emqx_json:decode(Payload, [return_maps])). + <<"clean_start">> := true, + <<"expiry_interval">> := 0, + <<"keepalive">> := 60 + }, emqx_json:decode(Payload, [return_maps])). recv_subscribed(_ClientId) -> {ok, #{qos := ?QOS_0, topic := Topic}} = receive_publish(100), - ?assertMatch(<<"$event/session_subscribed">>, Topic). + ?assertMatch(<<"$event/client_subscribed">>, Topic). recv_message_dropped(_ClientId) -> {ok, #{qos := ?QOS_0, topic := Topic}} = receive_publish(100), @@ -123,7 +123,7 @@ recv_message_acked(_ClientId) -> recv_unsubscribed(_ClientId) -> {ok, #{qos := ?QOS_0, topic := Topic}} = receive_publish(100), - ?assertMatch(<<"$event/session_unsubscribed">>, Topic). + ?assertMatch(<<"$event/client_unsubscribed">>, Topic). recv_disconnected(ClientId) -> {ok, #{qos := ?QOS_0, topic := Topic, payload := Payload}} = receive_publish(100), From 213b7c2501409d066b6e13133b648143af904e3c Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 18 Aug 2021 18:53:53 +0800 Subject: [PATCH 04/20] fix(config_api): improve the schema for config update APIs --- apps/emqx/src/emqx_map_lib.erl | 1 + .../src/emqx_mgmt_api_configs.erl | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl index cda4f3a85..e5baeb850 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx/src/emqx_map_lib.erl @@ -24,6 +24,7 @@ , safe_atom_key_map/1 , unsafe_atom_key_map/1 , jsonable_map/1 + , jsonable_value/1 , deep_convert/2 ]). diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index b54e357d6..6ff401048 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -24,6 +24,8 @@ , config_reset/2 ]). +-export([get_conf_schema/2]). + -define(PARAM_CONF_PATH, [#{ name => conf_path, in => query, @@ -171,15 +173,19 @@ get_conf_schema(BasePath, [{Key, Conf} | Confs], Result, MaxDepth) -> %% TODO: generate from hocon schema gen_schema(Conf) when is_boolean(Conf) -> - #{type => boolean}; + with_default_value(#{type => boolean}, Conf); gen_schema(Conf) when is_binary(Conf); is_atom(Conf) -> - #{type => string}; + with_default_value(#{type => string}, Conf); gen_schema(Conf) when is_number(Conf) -> - #{type => number}; + with_default_value(#{type => number}, Conf); gen_schema(Conf) when is_list(Conf) -> #{type => array, items => case Conf of [] -> #{}; %% don't know the type - _ -> gen_schema(hd(Conf)) + _ -> + case io_lib:printable_unicode_list(Conf) of + true -> gen_schema(unicode:characters_to_binary(Conf)); + false -> gen_schema(hd(Conf)) + end end}; gen_schema(Conf) when is_map(Conf) -> #{type => object, properties => @@ -189,6 +195,9 @@ gen_schema(_Conf) -> %% by the hocon schema #{type => string}. +with_default_value(Type, Value) -> + Type#{example => emqx_map_lib:jsonable_value(Value)}. + path_join(Path) -> path_join(Path, "/"). From 9c76247cdee45b7e1f55f06199eaac51c5f6333c Mon Sep 17 00:00:00 2001 From: DDDHuang <904897578@qq.com> Date: Thu, 19 Aug 2021 09:56:32 +0800 Subject: [PATCH 05/20] fix: alarms api timestamp millisecond --- apps/emqx/src/emqx_alarm.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index e52328d66..7599f9569 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -161,7 +161,7 @@ format(#activated_alarm{name = Name, message = Message, activate_at = At, detail node => node(), name => Name, message => Message, - duration => Now - At, + duration => (Now - At) div 1000, %% to millisecond details => Details }; format(#deactivated_alarm{name = Name, message = Message, activate_at = At, details = Details, From adc6226eae9a3bb62d9e062db7d4bd07ff261b32 Mon Sep 17 00:00:00 2001 From: lafirest Date: Wed, 18 Aug 2021 12:05:19 +0800 Subject: [PATCH 06/20] refactor(emqx_retainer): emqx_retainer_api use openapi model --- .../src/emqx_mgmt_api_configs.erl | 2 +- apps/emqx_retainer/src/emqx_retainer.erl | 13 +- apps/emqx_retainer/src/emqx_retainer_api.erl | 231 +++++++++++++++--- .../src/emqx_retainer_mnesia.erl | 58 +++-- .../test/emqx_retainer_api_SUITE.erl | 17 +- 5 files changed, 256 insertions(+), 65 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 6ff401048..1a89835ff 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -24,7 +24,7 @@ , config_reset/2 ]). --export([get_conf_schema/2]). +-export([get_conf_schema/2, gen_schema/1]). -define(PARAM_CONF_PATH, [#{ name => conf_path, diff --git a/apps/emqx_retainer/src/emqx_retainer.erl b/apps/emqx_retainer/src/emqx_retainer.erl index dfbe5cc69..4c36cb541 100644 --- a/apps/emqx_retainer/src/emqx_retainer.erl +++ b/apps/emqx_retainer/src/emqx_retainer.erl @@ -37,7 +37,8 @@ -export([ get_expiry_time/1 , update_config/1 , clean/0 - , delete/1]). + , delete/1 + , page_read/3]). %% gen_server callbacks -export([ init/1 @@ -66,6 +67,8 @@ -callback delete_message(context(), topic()) -> ok. -callback store_retained(context(), message()) -> ok. -callback read_message(context(), topic()) -> {ok, list()}. +-callback page_read(context(), topic(), non_neg_integer(), non_neg_integer()) -> + {ok, list()}. -callback match_messages(context(), topic(), cursor()) -> {ok, list(), cursor()}. -callback clear_expired(context()) -> ok. -callback clean(context()) -> ok. @@ -166,6 +169,9 @@ clean() -> delete(Topic) -> gen_server:call(?MODULE, {?FUNCTION_NAME, Topic}). +page_read(Topic, Page, Limit) -> + gen_server:call(?MODULE, {?FUNCTION_NAME, Topic, Page, Limit}). + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- @@ -198,6 +204,11 @@ handle_call({delete, Topic}, _, #{context := Context} = State) -> delete_message(Context, Topic), {reply, ok, State}; +handle_call({page_read, Topic, Page, Limit}, _, #{context := Context} = State) -> + Mod = get_backend_module(), + Result = Mod:page_read(Context, Topic, Page, Limit), + {reply, Result, State}; + handle_call(Req, _From, State) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, State}. diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index a0b72c858..d766eab06 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -16,52 +16,213 @@ -module(emqx_retainer_api). --rest_api(#{name => lookup_config, - method => 'GET', - path => "/retainer", - func => lookup_config, - descr => "lookup retainer config" - }). +-behaviour(minirest_api). --rest_api(#{name => update_config, - method => 'PUT', - path => "/retainer", - func => update_config, - descr => "update retainer config" - }). +-include_lib("emqx/include/emqx.hrl"). --export([ lookup_config/2 - , update_config/2 - ]). +-export([api_spec/0]). -lookup_config(_Bindings, _Params) -> - Config = emqx:get_config([emqx_retainer]), - return({ok, Config}). +-export([ lookup_retained_warp/2 + , with_topic_warp/2 + , config/2]). -update_config(_Bindings, Params) -> +-import(emqx_mgmt_api_configs, [gen_schema/1]). +-import(emqx_mgmt_util, [ response_array_schema/2 + , response_schema/1 + , response_error_schema/2]). + +-define(CFG_BODY(DESCR), + #{description => list_to_binary(DESCR), + content => #{<<"application/json">> => + #{schema => gen_schema(emqx_config:get([emqx_retainer]))}}}). + +api_spec() -> + { + [ lookup_retained_api() + , with_topic_api() + , config_api() + ], + [ message_schema(message, fun message_properties/0) + , message_schema(detail_message, fun detail_message_properties/0) + ] + }. + +lookup_retained_api() -> + Metadata = + #{get => #{description => <<"lookup matching messages">>, + parameters => [ #{name => page, + in => query, + description => <<"Page">>, + schema => #{type => integer, default => 1}} + , #{name => limit, + in => query, + description => <<"Page size">>, + schema => #{type => integer, + default => emqx_mgmt:max_row_limit()}} + ], + responses => #{ <<"200">> => + response_array_schema("List retained messages", message) + , <<"405">> => response_schema(<<"NotAllowed">>) + }}}, + {"/mqtt/retainer/messages", Metadata, lookup_retained_warp}. + +with_topic_api() -> + MetaData = #{get => #{description => <<"lookup matching messages">>, + parameters => [ #{name => topic, + in => path, + required => true, + schema => #{type => "string"}} + , #{name => page, + in => query, + description => <<"Page">>, + schema => #{type => integer, default => 1}} + , #{name => limit, + in => query, + description => <<"Page size">>, + schema => #{type => integer, + default => emqx_mgmt:max_row_limit()}} + ], + responses => #{ <<"200">> => + response_array_schema("List retained messages", detail_message) + , <<"405">> => response_schema(<<"NotAllowed">>)}}, + delete => #{description => <<"delete matching messages">>, + parameters => [#{name => topic, + in => path, + required => true, + schema => #{type => "string"}}], + responses => #{ <<"200">> => response_schema(<<"Successed">>) + , <<"405">> => response_schema(<<"NotAllowed">>)}} + }, + {"/mqtt/retainer/message/:topic", MetaData, with_topic_warp}. + +config_api() -> + MetaData = #{ + get => #{ + description => <<"get retainer config">>, + responses => #{<<"200">> => ?CFG_BODY("Get configs successfully"), + <<"404">> => response_error_schema( + <<"Config not found">>, ['NOT_FOUND'])} + }, + put => #{ + description => <<"Update retainer config">>, + 'requestBody' => + ?CFG_BODY("The format of the request body is depend on the 'conf_path' parameter in the query string"), + responses => #{<<"200">> => response_schema("Update configs successfully"), + <<"400">> => response_error_schema( + <<"Update configs failed">>, ['UPDATE_FAILED'])} + } + }, + {"/mqtt/retainer", MetaData, config}. + +lookup_retained_warp(Type, Req) -> + check_backend(Type, Req, fun lookup_retained/2). + +with_topic_warp(Type, Req) -> + check_backend(Type, Req, fun with_topic/2). + +config(get, _) -> + Config = emqx_config:get([emqx_retainer]), + Body = emqx_json:encode(Config), + {200, Body}; + +config(put, Req) -> try - ConfigList = proplists:get_value(<<"emqx_retainer">>, Params), - {ok, RawConf} = hocon:binary(jsx:encode(#{<<"emqx_retainer">> => ConfigList}), + {ok, Body, _} = cowboy_req:read_body(Req), + Cfg = emqx_json:decode(Body), + {ok, RawConf} = hocon:binary(jsx:encode(#{<<"emqx_retainer">> => Cfg}), #{format => richmap}), RichConf = hocon_schema:check(emqx_retainer_schema, RawConf, #{atom_key => true}), #{emqx_retainer := Conf} = hocon_schema:richmap_to_map(RichConf), - Action = proplists:get_value(<<"action">>, Params, undefined), - do_update_config(Action, Conf), - return() - catch _:_:Reason -> - return({error, Reason}) + emqx_retainer:update_config(Conf), + {200, #{<<"content-type">> => <<"text/plain">>}, <<"Update configs successfully">>} + catch _:Reason:_ -> + {400, + #{code => 'UPDATE_FAILED', + message => erlang:list_to_binary(io_lib:format("~p~n", [Reason]))}} end. %%------------------------------------------------------------------------------ %% Interval Funcs %%------------------------------------------------------------------------------ -do_update_config(undefined, Config) -> - emqx_retainer:update_config(Config); -do_update_config(<<"test">>, _) -> - ok. +lookup_retained(get, Req) -> + lookup(undefined, Req, fun format_message/1). -%% TODO: V5 API -return() -> - ok. -return(_) -> - ok. +with_topic(get, Req) -> + Topic = cowboy_req:binding(topic, Req), + lookup(Topic, Req, fun format_detail_message/1); + +with_topic(delete, Req) -> + Topic = cowboy_req:binding(topic, Req), + emqx_retainer_mnesia:delete_message(undefined, Topic), + {200}. + +-spec lookup(undefined | binary(), + cowboy_req:req(), + fun((#message{}) -> map())) -> + {200, map()}. +lookup(Topic, Req, Formatter) -> + #{page := Page, + limit := Limit} = cowboy_req:match_qs([{page, int, 1}, + {limit, int, emqx_mgmt:max_row_limit()}], + Req), + {ok, Msgs} = emqx_retainer_mnesia:page_read(undefined, Topic, Page, Limit), + {200, format_message(Msgs, Formatter)}. + + +message_schema(Type, Properties) -> + #{Type => #{type => object, + properties => Properties()}}. + +message_properties() -> + #{msgid => #{type => string, + description => <<"Message ID">>}, + topic => #{type => string, + description => <<"Topic">>}, + qos => #{type => integer, + enum => [0, 1, 2], + description => <<"Qos">>}, + publish_at => #{type => string, + description => <<"publish datetime">>}, + from_clientid => #{type => string, + description => <<"Message from">>}, + from_username => #{type => string, + description => <<"publish username">>}}. + +detail_message_properties() -> + Base = message_properties(), + Base#{payload => #{type => string, + description => <<"Topic">>}}. + +format_message(Messages, Formatter) when is_list(Messages)-> + [Formatter(Message) || Message <- Messages]; + +format_message(Message, Formatter) -> + Formatter(Message). + +format_message(#message{id = ID, qos = Qos, topic = Topic, from = From, timestamp = Timestamp, headers = Headers}) -> + #{msgid => emqx_guid:to_hexstr(ID), + qos => Qos, + topic => Topic, + publish_at => erlang:list_to_binary(emqx_mgmt_util:strftime(Timestamp div 1000)), + from_clientid => to_bin_string(From), + from_username => maps:get(username, Headers, <<>>) + }. + +format_detail_message(#message{payload = Payload} = Msg) -> + Base = format_message(Msg), + Base#{payload => Payload}. + +to_bin_string(Data) when is_binary(Data) -> + Data; +to_bin_string(Data) -> + list_to_binary(io_lib:format("~p", [Data])). + +check_backend(Type, Req, Cont) -> + case emqx:get_config([emqx_retainer, config, type]) of + built_in_database -> + Cont(Type, Req); + _ -> + {405, + #{<<"content-type">> => <<"text/plain">>}, + <<"This API only for built in database">>} + end. diff --git a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl index 1c9956050..5f91a40c9 100644 --- a/apps/emqx_retainer/src/emqx_retainer_mnesia.erl +++ b/apps/emqx_retainer/src/emqx_retainer_mnesia.erl @@ -24,9 +24,10 @@ -include_lib("stdlib/include/qlc.hrl"). --export([delete_message/2 +-export([ delete_message/2 , store_retained/2 , read_message/2 + , page_read/4 , match_messages/3 , clear_expired/1 , clean/1]). @@ -129,6 +130,19 @@ delete_message(_, Topic) -> read_message(_, Topic) -> {ok, read_messages(Topic)}. +page_read(_, Topic, Page, Limit) -> + Cursor = make_cursor(Topic), + case Page > 1 of + true -> + _ = qlc:next_answers(Cursor, (Page - 1) * Limit), + ok; + _ -> + ok + end, + Rows = qlc:next_answers(Cursor, Limit), + qlc:delete_cursor(Cursor), + {ok, Rows}. + match_messages(_, Topic, Cursor) -> MaxReadNum = emqx:get_config([?APP, flow_control, max_read_number]), case Cursor of @@ -152,34 +166,28 @@ clean(_) -> sort_retained([]) -> []; sort_retained([Msg]) -> [Msg]; sort_retained(Msgs) -> - lists:sort(fun(#message{timestamp = Ts1}, #message{timestamp = Ts2}) -> - Ts1 =< Ts2 end, - Msgs). + lists:sort(fun compare_message/2, Msgs). + +compare_message(M1, M2) -> + M1#message.timestamp =< M2#message.timestamp. -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- topic2tokens(Topic) -> emqx_topic:words(Topic). -spec start_batch_read(topic(), pos_integer()) -> batch_read_result(). start_batch_read(Topic, MaxReadNum) -> - Ms = make_match_spec(Topic), - TabQH = ets:table(?TAB, [{traverse, {select, Ms}}]), - QH = qlc:q([E || E <- TabQH]), - Cursor = qlc:cursor(QH), + Cursor = make_cursor(Topic), batch_read_messages(Cursor, MaxReadNum). -spec batch_read_messages(emqx_retainer_storage:cursor(), pos_integer()) -> batch_read_result(). batch_read_messages(Cursor, MaxReadNum) -> Answers = qlc:next_answers(Cursor, MaxReadNum), - Orders = sort_retained(Answers), - case erlang:length(Orders) < MaxReadNum of + case erlang:length(Answers) < MaxReadNum of true -> qlc:delete_cursor(Cursor), - {ok, Orders, undefined}; + {ok, Answers, undefined}; _ -> - {ok, Orders, Cursor} + {ok, Answers, Cursor} end. -spec(read_messages(emqx_types:topic()) @@ -217,14 +225,28 @@ condition(Ws) -> _ -> (Ws1 -- ['#']) ++ '_' end. --spec make_match_spec(topic()) -> ets:match_spec(). -make_match_spec(Filter) -> +-spec make_match_spec(undefined | topic()) -> ets:match_spec(). +make_match_spec(Topic) -> NowMs = erlang:system_time(millisecond), - Cond = condition(emqx_topic:words(Filter)), + Cond = + case Topic of + undefined -> + '_'; + _ -> + condition(emqx_topic:words(Topic)) + end, MsHd = #retained{topic = Cond, msg = '$2', expiry_time = '$3'}, [{MsHd, [{'=:=', '$3', 0}], ['$2']}, {MsHd, [{'>', '$3', NowMs}], ['$2']}]. +-spec make_cursor(undefined | topic()) -> qlc:query_cursor(). +make_cursor(Topic) -> + Ms = make_match_spec(Topic), + TabQH = ets:table(?TAB, [{traverse, {select, Ms}}]), + QH = qlc:q([E || E <- TabQH]), + QH2 = qlc:sort(QH, {order, fun compare_message/2}), + qlc:cursor(QH2). + -spec is_table_full() -> boolean(). is_table_full() -> #{max_retained_messages := Limit} = emqx:get_config([?APP, config]), diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index 6ce64ae2e..5d379e8cd 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -33,10 +33,11 @@ -define(HOST, "http://127.0.0.1:8081/"). -define(API_VERSION, "v4"). -define(BASE_PATH, "api"). +-define(CFG_URI, "/configs/retainer"). all() -> -%% TODO: V5 API -%% emqx_ct:all(?MODULE). + %% TODO: V5 API + %% emqx_ct:all(?MODULE). []. groups() -> @@ -69,16 +70,12 @@ set_special_configs(_) -> %%------------------------------------------------------------------------------ t_config(_Config) -> - {ok, Return} = request_http_rest_lookup(["retainer"]), + {ok, Return} = request_http_rest_lookup([?CFG_URI]), NowCfg = get_http_data(Return), NewCfg = NowCfg#{<<"msg_expiry_interval">> => timer:seconds(60)}, RetainerConf = #{<<"emqx_retainer">> => NewCfg}, - {ok, _} = request_http_rest_update(["retainer?action=test"], RetainerConf), - {ok, TestReturn} = request_http_rest_lookup(["retainer"]), - ?assertEqual(NowCfg, get_http_data(TestReturn)), - - {ok, _} = request_http_rest_update(["retainer"], RetainerConf), + {ok, _} = request_http_rest_update([?CFG_URI], RetainerConf), {ok, UpdateReturn} = request_http_rest_lookup(["retainer"]), ?assertEqual(NewCfg, get_http_data(UpdateReturn)), ok. @@ -141,12 +138,12 @@ receive_messages(Count, Msgs) -> end. switch_emqx_retainer(undefined, IsEnable) -> - {ok, Return} = request_http_rest_lookup(["retainer"]), + {ok, Return} = request_http_rest_lookup([?COMMON_SHARD]), NowCfg = get_http_data(Return), switch_emqx_retainer(NowCfg, IsEnable); switch_emqx_retainer(NowCfg, IsEnable) -> NewCfg = NowCfg#{<<"enable">> => IsEnable}, RetainerConf = #{<<"emqx_retainer">> => NewCfg}, - {ok, _} = request_http_rest_update(["retainer"], RetainerConf), + {ok, _} = request_http_rest_update([?CFG_URI], RetainerConf), NewCfg. From e17612b2377b25808494b0da982550390ad4a387 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 19 Aug 2021 09:45:14 +0800 Subject: [PATCH 07/20] fix(config): return only updated confs for emqx:update_config/2,3 --- apps/emqx/src/emqx_config_handler.erl | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index 2ee90cb04..a7c28fff0 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -89,8 +89,8 @@ handle_call({add_child, ConfKeyPath, HandlerName}, _From, handle_call({change_config, SchemaModule, ConfKeyPath, UpdateArgs}, _From, #{handlers := Handlers} = State) -> - OldConf = emqx_config:get([]), - OldRawConf = emqx_config:get_raw([]), + OldConf = emqx_config:get_root(ConfKeyPath), + OldRawConf = emqx_config:get_root_raw(ConfKeyPath), Reply = try case process_update_request(ConfKeyPath, OldRawConf, Handlers, UpdateArgs) of {ok, NewRawConf, OverrideConf} -> @@ -151,7 +151,8 @@ check_and_save_configs(SchemaModule, ConfKeyPath, Handlers, NewRawConf, OldConf, {AppEnvs, CheckedConf} = emqx_config:check_config(SchemaModule, NewRawConf), case do_post_config_update(ConfKeyPath, Handlers, OldConf, CheckedConf, UpdateArgs, #{}) of {ok, Result0} -> - case save_configs(AppEnvs, CheckedConf, NewRawConf, OverrideConf, UpdateArgs) of + case save_configs(ConfKeyPath, AppEnvs, CheckedConf, NewRawConf, OverrideConf, + UpdateArgs) of {ok, Result1} -> {ok, Result1#{post_config_update => Result0}}; Error -> Error @@ -200,9 +201,10 @@ call_post_config_update(Handlers, OldConf, NewConf, UpdateReq, Result) -> false -> {ok, Result} end. -save_configs(AppEnvs, CheckedConf, NewRawConf, OverrideConf, {_Cmd, Opts}) -> +save_configs(ConfKeyPath, AppEnvs, CheckedConf, NewRawConf, OverrideConf, {_Cmd, Opts}) -> case emqx_config:save_configs(AppEnvs, CheckedConf, NewRawConf, OverrideConf) of - ok -> {ok, #{config => emqx_config:get([]), raw_config => return_rawconf(Opts)}}; + ok -> {ok, #{config => emqx_config:get(ConfKeyPath), + raw_config => return_rawconf(ConfKeyPath, Opts)}}; {error, Reason} -> {error, {save_configs, Reason}} end. @@ -223,10 +225,11 @@ update_override_config(RawConf) -> up_req({remove, _Opts}) -> '$remove'; up_req({{update, Req}, _Opts}) -> Req. -return_rawconf(#{rawconf_with_defaults := true}) -> - emqx_config:fill_defaults(emqx_config:get_raw([])); -return_rawconf(_) -> - emqx_config:get_raw([]). +return_rawconf(ConfKeyPath, #{rawconf_with_defaults := true}) -> + FullRawConf = emqx_config:fill_defaults(emqx_config:get_raw([])), + emqx_map_lib:deep_get(bin_path(ConfKeyPath), FullRawConf); +return_rawconf(ConfKeyPath, _) -> + emqx_config:get_raw(ConfKeyPath). bin_path(ConfKeyPath) -> [bin(Key) || Key <- ConfKeyPath]. From ef59309ed0b7a8d183b521f28ce4dfc2ac85a943 Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Thu, 19 Aug 2021 15:29:34 +0800 Subject: [PATCH 08/20] fix(config): check config failed when updating --- apps/emqx/src/emqx_config_handler.erl | 26 ++++++++++++------- apps/emqx/src/emqx_map_lib.erl | 2 +- .../src/emqx_mgmt_api_configs.erl | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index a7c28fff0..7c66656ce 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -89,12 +89,10 @@ handle_call({add_child, ConfKeyPath, HandlerName}, _From, handle_call({change_config, SchemaModule, ConfKeyPath, UpdateArgs}, _From, #{handlers := Handlers} = State) -> - OldConf = emqx_config:get_root(ConfKeyPath), - OldRawConf = emqx_config:get_root_raw(ConfKeyPath), Reply = try - case process_update_request(ConfKeyPath, OldRawConf, Handlers, UpdateArgs) of + case process_update_request(ConfKeyPath, Handlers, UpdateArgs) of {ok, NewRawConf, OverrideConf} -> - check_and_save_configs(SchemaModule, ConfKeyPath, Handlers, NewRawConf, OldConf, + check_and_save_configs(SchemaModule, ConfKeyPath, Handlers, NewRawConf, OverrideConf, UpdateArgs); {error, Result} -> {error, Result} @@ -121,12 +119,14 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -process_update_request(ConfKeyPath, OldRawConf, _Handlers, {remove, _Opts}) -> +process_update_request(ConfKeyPath, _Handlers, {remove, _Opts}) -> + OldRawConf = emqx_config:get_root_raw(ConfKeyPath), BinKeyPath = bin_path(ConfKeyPath), NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), OverrideConf = emqx_map_lib:deep_remove(BinKeyPath, emqx_config:read_override_conf()), {ok, NewRawConf, OverrideConf}; -process_update_request(ConfKeyPath, OldRawConf, Handlers, {{update, UpdateReq}, _Opts}) -> +process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, _Opts}) -> + OldRawConf = emqx_config:get_root_raw(ConfKeyPath), case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of {ok, NewRawConf} -> OverrideConf = update_override_config(NewRawConf), @@ -146,12 +146,15 @@ do_update_config([ConfKey | ConfKeyPath], Handlers, OldRawConf, UpdateReq) -> Error end. -check_and_save_configs(SchemaModule, ConfKeyPath, Handlers, NewRawConf, OldConf, OverrideConf, +check_and_save_configs(SchemaModule, ConfKeyPath, Handlers, NewRawConf, OverrideConf, UpdateArgs) -> - {AppEnvs, CheckedConf} = emqx_config:check_config(SchemaModule, NewRawConf), - case do_post_config_update(ConfKeyPath, Handlers, OldConf, CheckedConf, UpdateArgs, #{}) of + OldConf = emqx_config:get_root(ConfKeyPath), + FullRawConf = with_full_raw_confs(NewRawConf), + {AppEnvs, CheckedConf} = emqx_config:check_config(SchemaModule, FullRawConf), + NewConf = maps:with(maps:keys(OldConf), CheckedConf), + case do_post_config_update(ConfKeyPath, Handlers, OldConf, NewConf, UpdateArgs, #{}) of {ok, Result0} -> - case save_configs(ConfKeyPath, AppEnvs, CheckedConf, NewRawConf, OverrideConf, + case save_configs(ConfKeyPath, AppEnvs, NewConf, NewRawConf, OverrideConf, UpdateArgs) of {ok, Result1} -> {ok, Result1#{post_config_update => Result0}}; @@ -231,6 +234,9 @@ return_rawconf(ConfKeyPath, #{rawconf_with_defaults := true}) -> return_rawconf(ConfKeyPath, _) -> emqx_config:get_raw(ConfKeyPath). +with_full_raw_confs(PartialConf) -> + maps:merge(emqx_config:get_raw([]), PartialConf). + bin_path(ConfKeyPath) -> [bin(Key) || Key <- ConfKeyPath]. bin(A) when is_atom(A) -> atom_to_binary(A, utf8); diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl index e5baeb850..468553193 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx/src/emqx_map_lib.erl @@ -118,7 +118,7 @@ unsafe_atom_key_map(Map) -> safe_atom_key_map(Map) -> covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end). --spec jsonable_map(map()) -> map(). +-spec jsonable_map(map() | list()) -> map() | list(). jsonable_map(Map) -> deep_convert(Map, fun(K, V) -> {jsonable_value(K), jsonable_value(V)} diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 1a89835ff..46b679c3d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -120,7 +120,7 @@ config(put, Req) -> Path = conf_path(Req), {ok, #{raw_config := RawConf}} = emqx:update_config(Path, http_body(Req), #{rawconf_with_defaults => true}), - {200, emqx_map_lib:deep_get(Path, emqx_map_lib:jsonable_map(RawConf))}. + {200, emqx_map_lib:jsonable_map(RawConf)}. config_reset(post, Req) -> %% reset the config specified by the query string param 'conf_path' From 61da3a4fd70e7716dc7022347eae9d13306db69c Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 10 Aug 2021 14:47:22 +0800 Subject: [PATCH 09/20] feat(authn hot config): initial support for hot config --- apps/emqx_authn/src/emqx_authn.erl | 102 +++++++++++++++++- apps/emqx_authn/src/emqx_authn_api.erl | 48 +++------ apps/emqx_authn/src/emqx_authn_app.erl | 1 + ...hema.erl => emqx_authn_implied_schema.erl} | 2 +- 4 files changed, 118 insertions(+), 35 deletions(-) rename apps/emqx_authn/src/simple_authn/{emqx_authn_other_schema.erl => emqx_authn_implied_schema.erl} (97%) diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 034e06b89..7ead53638 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -16,8 +16,17 @@ -module(emqx_authn). +-behaviour(emqx_config_handler). + -include("emqx_authn.hrl"). +-export([mnesia/1]). + +-export([ pre_config_update/2 + , post_config_update/3 + , update_config/2 + ]). + -export([ enable/0 , disable/0 , is_enabled/0 @@ -46,8 +55,6 @@ , list_users/2 ]). --export([mnesia/1]). - -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). @@ -75,6 +82,97 @@ mnesia(boot) -> mnesia(copy) -> ok = ekka_mnesia:copy_table(?CHAIN_TAB, ram_copies). +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +pre_config_update({enable, Enable}, _OldConfig) -> + Enable; +pre_config_update({create_authenticator, Config}, OldConfig) -> + OldConfig ++ [Config]; +pre_config_update({delete_authenticator, ID}, OldConfig) -> + case lookup_authenticator(?CHAIN, ID) of + {error, Reason} -> error(Reason); + {ok, #{name := Name}} -> + lists:filter(fun(#{<<"name">> := N}) -> + N =/= Name + end, OldConfig) + end; +pre_config_update({update_authenticator, ID, Config}, OldConfig) -> + case lookup_authenticator(?CHAIN, ID) of + {error, Reason} -> error(Reason); + {ok, #{name := Name}} -> + lists:map(fun(#{<<"name">> := N} = C) -> + case N =:= Name of + true -> Config; + false -> C + end + end, OldConfig) + end; +pre_config_update({update_or_create_authenticator, ID, Config}, OldConfig) -> + case lookup_authenticator(?CHAIN, ID) of + {error, _Reason} -> OldConfig ++ [Config]; + {ok, #{name := Name}} -> + lists:map(fun(#{<<"name">> := N} = C) -> + case N =:= Name of + true -> Config; + false -> C + end + end, OldConfig) + end. + +post_config_update({enable, true}, _NewConfig, _OldConfig) -> + emqx_authn:enable(); +post_config_update({enable, false}, _NewConfig, _OldConfig) -> + emqx_authn:disable(); +post_config_update({create_authenticator, #{<<"name">> := Name}}, NewConfig, _OldConfig) -> + case lists:filter( + fun(#{name := N}) -> + N =:= Name + end, NewConfig) of + [Config] -> + case create_authenticator(?CHAIN, Config) of + {ok, _} -> ok; + {error, Reason} -> throw(Reason) + end; + [_Config | _] -> + error(name_has_be_used) + end; +post_config_update({delete_authenticator, ID}, _NewConfig, _OldConfig) -> + case delete_authenticator(?CHAIN, ID) of + ok -> ok; + {error, Reason} -> throw(Reason) + end; +post_config_update({update_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig) -> + case lists:filter( + fun(#{name := N}) -> + N =:= Name + end, NewConfig) of + [Config] -> + case update_authenticator(?CHAIN, ID, Config) of + {ok, _} -> ok; + {error, Reason} -> throw(Reason) + end; + [_Config | _] -> + error(name_has_be_used) + end; +post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig) -> + case lists:filter( + fun(#{name := N}) -> + N =:= Name + end, NewConfig) of + [Config] -> + case update_or_create_authenticator(?CHAIN, ID, Config) of + {ok, _} -> ok; + {error, Reason} -> throw(Reason) + end; + [_Config | _] -> + error(name_has_be_used) + end. + +update_config(Path, ConfigRequest) -> + emqx_config:update(emqx_authn_schema, Path, ConfigRequest). + enable() -> case emqx:hook('client.authenticate', {?MODULE, authenticate, []}) of ok -> ok; diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 78ef5fd35..1e232d553 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1253,14 +1253,9 @@ definitions() -> authentication(post, Request) -> {ok, Body, _} = cowboy_req:read_body(Request), case emqx_json:decode(Body, [return_maps]) of - #{<<"enable">> := true} -> - ok = emqx_authn:enable(), + #{<<"enable">> := Enable} -> + emqx_authn:update_config([authentication, enable], {enable, Enable}), {204}; - #{<<"enable">> := false} -> - ok = emqx_authn:disable(), - {204}; - #{<<"enable">> := _} -> - serialize_error({invalid_parameter, enable}); _ -> serialize_error({missing_parameter, enable}) end; @@ -1270,16 +1265,10 @@ authentication(get, _Request) -> authenticators(post, Request) -> {ok, Body, _} = cowboy_req:read_body(Request), - AuthenticatorConfig = emqx_json:decode(Body, [return_maps]), - Config = #{<<"authentication">> => #{ - <<"authenticators">> => [AuthenticatorConfig] - }}, - NConfig = hocon_schema:check_plain(emqx_authn_schema, Config, - #{nullable => true}), - #{authentication := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig), - case emqx_authn:create_authenticator(?CHAIN, NAuthenticatorConfig) of - {ok, Authenticator2} -> - {201, Authenticator2}; + Config = emqx_json:decode(Body, [return_maps]), + case emqx_authn:update_config([authentication, authenticators], {create_authenticator, Config}) of + ok -> + {204}; {error, Reason} -> serialize_error(Reason) end; @@ -1298,22 +1287,17 @@ authenticators2(get, Request) -> authenticators2(put, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), {ok, Body, _} = cowboy_req:read_body(Request), - AuthenticatorConfig = emqx_json:decode(Body, [return_maps]), - Config = #{<<"authentication">> => #{ - <<"authenticators">> => [AuthenticatorConfig] - }}, - NConfig = hocon_schema:check_plain(emqx_authn_schema, Config, - #{nullable => true}), - #{authentication := #{authenticators := [NAuthenticatorConfig]}} = emqx_map_lib:unsafe_atom_key_map(NConfig), - case emqx_authn:update_or_create_authenticator(?CHAIN, AuthenticatorID, NAuthenticatorConfig) of - {ok, Authenticator} -> - {200, Authenticator}; + Config = emqx_json:decode(Body, [return_maps]), + case emqx_authn:update_config([authentication, authenticators], + {update_or_create_authenticator, AuthenticatorID, Config}) of + ok -> + {204}; {error, Reason} -> serialize_error(Reason) end; authenticators2(delete, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), - case emqx_authn:delete_authenticator(?CHAIN, AuthenticatorID) of + case emqx_authn:update_config([authentication, authenticators], {delete_authenticator, AuthenticatorID}) of ok -> {204}; {error, Reason} -> @@ -1324,7 +1308,7 @@ position(post, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), {ok, Body, _} = cowboy_req:read_body(Request), NBody = emqx_json:decode(Body, [return_maps]), - Config = hocon_schema:check_plain(emqx_authn_other_schema, #{<<"position">> => NBody}, + Config = hocon_schema:check_plain(emqx_authn_implied_schema, #{<<"position">> => NBody}, #{nullable => true}, ["position"]), #{position := #{position := Position}} = emqx_map_lib:unsafe_atom_key_map(Config), case emqx_authn:move_authenticator_to_the_nth(?CHAIN, AuthenticatorID, Position) of @@ -1338,7 +1322,7 @@ import_users(post, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), {ok, Body, _} = cowboy_req:read_body(Request), NBody = emqx_json:decode(Body, [return_maps]), - Config = hocon_schema:check_plain(emqx_authn_other_schema, #{<<"filename">> => NBody}, + Config = hocon_schema:check_plain(emqx_authn_implied_schema, #{<<"filename">> => NBody}, #{nullable => true}, ["filename"]), #{filename := #{filename := Filename}} = emqx_map_lib:unsafe_atom_key_map(Config), case emqx_authn:import_users(?CHAIN, AuthenticatorID, Filename) of @@ -1352,7 +1336,7 @@ users(post, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), {ok, Body, _} = cowboy_req:read_body(Request), NBody = emqx_json:decode(Body, [return_maps]), - Config = hocon_schema:check_plain(emqx_authn_other_schema, #{<<"user_info">> => NBody}, + Config = hocon_schema:check_plain(emqx_authn_implied_schema, #{<<"user_info">> => NBody}, #{nullable => true}, ["user_info"]), #{user_info := UserInfo} = emqx_map_lib:unsafe_atom_key_map(Config), case emqx_authn:add_user(?CHAIN, AuthenticatorID, UserInfo) of @@ -1375,7 +1359,7 @@ users2(patch, Request) -> UserID = cowboy_req:binding(user_id, Request), {ok, Body, _} = cowboy_req:read_body(Request), NBody = emqx_json:decode(Body, [return_maps]), - Config = hocon_schema:check_plain(emqx_authn_other_schema, #{<<"new_user_info">> => NBody}, + Config = hocon_schema:check_plain(emqx_authn_implied_schema, #{<<"new_user_info">> => NBody}, #{nullable => true}, ["new_user_info"]), #{new_user_info := NewUserInfo} = emqx_map_lib:unsafe_atom_key_map(Config), case emqx_authn:update_user(?CHAIN, AuthenticatorID, UserID, NewUserInfo) of diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index 7518e5a01..b7f409bc9 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -29,6 +29,7 @@ start(_StartType, _StartArgs) -> ok = ekka_rlog:wait_for_shards([?AUTH_SHARD], infinity), {ok, Sup} = emqx_authn_sup:start_link(), + emqx_config_handler:add_handler([authentication, authenticators], emqx_authn), initialize(), {ok, Sup}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_other_schema.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_implied_schema.erl similarity index 97% rename from apps/emqx_authn/src/simple_authn/emqx_authn_other_schema.erl rename to apps/emqx_authn/src/simple_authn/emqx_authn_implied_schema.erl index 0f5c8abb8..1a0731e92 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_other_schema.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_implied_schema.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_authn_other_schema). +-module(emqx_authn_implied_schema). -include("emqx_authn.hrl"). -include_lib("typerefl/include/types.hrl"). From 60f0e8e5a5bb305588482888c529f76f243c066e Mon Sep 17 00:00:00 2001 From: zhouzb Date: Tue, 10 Aug 2021 17:04:20 +0800 Subject: [PATCH 10/20] refactor(authn): replace mnesia with ets table --- apps/emqx_authn/src/emqx_authn.erl | 465 ++++++++++++++----------- apps/emqx_authn/src/emqx_authn_sup.erl | 9 +- 2 files changed, 273 insertions(+), 201 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 7ead53638..703b0efcf 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -16,11 +16,12 @@ -module(emqx_authn). +-behaviour(gen_server). + -behaviour(emqx_config_handler). -include("emqx_authn.hrl"). - --export([mnesia/1]). +-include_lib("emqx/include/logger.hrl"). -export([ pre_config_update/2 , post_config_update/3 @@ -34,6 +35,10 @@ -export([authenticate/2]). +-export([ start_link/0 + , stop/0 + ]). + -export([ create_chain/1 , delete_chain/1 , lookup_chain/1 @@ -55,33 +60,17 @@ , list_users/2 ]). --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). -define(CHAIN_TAB, emqx_authn_chain). --rlog_shard({?AUTH_SHARD, ?CHAIN_TAB}). - -%%------------------------------------------------------------------------------ -%% Mnesia bootstrap -%%------------------------------------------------------------------------------ - -%% @doc Create or replicate tables. --spec(mnesia(boot) -> ok). -mnesia(boot) -> - %% Optimize storage - StoreProps = [{ets, [{read_concurrency, true}]}], - %% Chain table - ok = ekka_mnesia:create_table(?CHAIN_TAB, [ - {ram_copies, [node()]}, - {record_name, chain}, - {local_content, true}, - {attributes, record_info(fields, chain)}, - {storage_properties, StoreProps}]); - -mnesia(copy) -> - ok = ekka_mnesia:copy_table(?CHAIN_TAB, ram_copies). - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ @@ -192,7 +181,7 @@ is_enabled() -> end, Callbacks). authenticate(Credential, _AuthResult) -> - case mnesia:dirty_read(?CHAIN_TAB, ?CHAIN) of + case ets:lookup(?CHAIN_TAB, ?CHAIN) of [#chain{authenticators = Authenticators}] -> do_authenticate(Authenticators, Credential); [] -> @@ -214,154 +203,39 @@ do_authenticate([{_, _, #authenticator{provider = Provider, state = State}} | Mo {stop, Result} end. +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +stop() -> + gen_server:stop(?MODULE). + create_chain(#{id := ID}) -> - trans( - fun() -> - case mnesia:read(?CHAIN_TAB, ID, write) of - [] -> - Chain = #chain{id = ID, - authenticators = [], - created_at = erlang:system_time(millisecond)}, - mnesia:write(?CHAIN_TAB, Chain, write), - {ok, serialize_chain(Chain)}; - [_ | _] -> - {error, {already_exists, {chain, ID}}} - end - end). + gen_server:call(?MODULE, {create_chain, ID}). delete_chain(ID) -> - trans( - fun() -> - case mnesia:read(?CHAIN_TAB, ID, write) of - [] -> - {error, {not_found, {chain, ID}}}; - [#chain{authenticators = Authenticators}] -> - _ = [do_delete_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators], - mnesia:delete(?CHAIN_TAB, ID, write) - end - end). + gen_server:call(?MODULE, {delete_chain, ID}). lookup_chain(ID) -> - case mnesia:dirty_read(?CHAIN_TAB, ID) of - [] -> - {error, {not_found, {chain, ID}}}; - [Chain] -> - {ok, serialize_chain(Chain)} - end. + gen_server:call(?MODULE, {lookup_chain, ID}). list_chains() -> Chains = ets:tab2list(?CHAIN_TAB), {ok, [serialize_chain(Chain) || Chain <- Chains]}. -create_authenticator(ChainID, #{name := Name} = Config) -> - UpdateFun = - fun(Chain = #chain{authenticators = Authenticators}) -> - case lists:keymember(Name, 2, Authenticators) of - true -> - {error, name_has_be_used}; - false -> - AlreadyExist = fun(ID) -> - lists:keymember(ID, 1, Authenticators) - end, - AuthenticatorID = gen_id(AlreadyExist), - case do_create_authenticator(ChainID, AuthenticatorID, Config) of - {ok, Authenticator} -> - NAuthenticators = Authenticators ++ [{AuthenticatorID, Name, Authenticator}], - ok = mnesia:write(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}, write), - {ok, serialize_authenticator(Authenticator)}; - {error, Reason} -> - {error, Reason} - end - end - end, - update_chain(ChainID, UpdateFun). +create_authenticator(ChainID, Config) -> + gen_server:call(?MODULE, {create_authenticator, ChainID, Config}). delete_authenticator(ChainID, AuthenticatorID) -> - UpdateFun = fun(Chain = #chain{authenticators = Authenticators}) -> - case lists:keytake(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {value, {_, _, Authenticator}, NAuthenticators} -> - _ = do_delete_authenticator(Authenticator), - NChain = Chain#chain{authenticators = NAuthenticators}, - mnesia:write(?CHAIN_TAB, NChain, write) - end - end, - update_chain(ChainID, UpdateFun). + gen_server:call(?MODULE, {delete_authenticator, ChainID, AuthenticatorID}). update_authenticator(ChainID, AuthenticatorID, Config) -> - do_update_authenticator(ChainID, AuthenticatorID, Config, false). + gen_server:call(?MODULE, {update_authenticator, ChainID, AuthenticatorID, Config}). update_or_create_authenticator(ChainID, AuthenticatorID, Config) -> - do_update_authenticator(ChainID, AuthenticatorID, Config, true). - -do_update_authenticator(ChainID, AuthenticatorID, #{name := NewName} = Config, CreateWhenNotFound) -> - UpdateFun = fun(Chain = #chain{authenticators = Authenticators}) -> - case lists:keytake(AuthenticatorID, 1, Authenticators) of - false -> - case CreateWhenNotFound of - true -> - case lists:keymember(NewName, 2, Authenticators) of - true -> - {error, name_has_be_used}; - false -> - case do_create_authenticator(ChainID, AuthenticatorID, Config) of - {ok, Authenticator} -> - NAuthenticators = Authenticators ++ [{AuthenticatorID, NewName, Authenticator}], - ok = mnesia:write(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}, write), - {ok, serialize_authenticator(Authenticator)}; - {error, Reason} -> - {error, Reason} - end - end; - false -> - {error, {not_found, {authenticator, AuthenticatorID}}} - end; - {value, - {_, _, #authenticator{provider = Provider, - state = #{version := Version} = State} = Authenticator}, - Others} -> - case lists:keymember(NewName, 2, Others) of - true -> - {error, name_has_be_used}; - false -> - case (NewProvider = authenticator_provider(Config)) =:= Provider of - true -> - Unique = <>, - case Provider:update(Config#{'_unique' => Unique}, State) of - {ok, NewState} -> - NewAuthenticator = Authenticator#authenticator{name = NewName, - config = Config, - state = switch_version(NewState)}, - NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), - ok = mnesia:write(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}, write), - {ok, serialize_authenticator(NewAuthenticator)}; - {error, Reason} -> - {error, Reason} - end; - false -> - Unique = <>, - case NewProvider:create(Config#{'_unique' => Unique}) of - {ok, NewState} -> - NewAuthenticator = Authenticator#authenticator{name = NewName, - provider = NewProvider, - config = Config, - state = switch_version(NewState)}, - NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), - ok = mnesia:write(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}, write), - _ = Provider:destroy(State), - {ok, serialize_authenticator(NewAuthenticator)}; - {error, Reason} -> - {error, Reason} - end - end - end - end - end, - update_chain(ChainID, UpdateFun). + gen_server:call(?MODULE, {update_or_create_authenticator, ChainID, AuthenticatorID, Config}). lookup_authenticator(ChainID, AuthenticatorID) -> - case mnesia:dirty_read(?CHAIN_TAB, ChainID) of + case ets:lookup(?CHAIN_TAB, ChainID) of [] -> {error, {not_found, {chain, ChainID}}}; [#chain{authenticators = Authenticators}] -> @@ -374,7 +248,7 @@ lookup_authenticator(ChainID, AuthenticatorID) -> end. list_authenticators(ChainID) -> - case mnesia:dirty_read(?CHAIN_TAB, ChainID) of + case ets:lookup(?CHAIN_TAB, ChainID) of [] -> {error, {not_found, {chain, ChainID}}}; [#chain{authenticators = Authenticators}] -> @@ -382,34 +256,172 @@ list_authenticators(ChainID) -> end. move_authenticator_to_the_nth(ChainID, AuthenticatorID, N) -> - UpdateFun = fun(Chain = #chain{authenticators = Authenticators}) -> - case move_authenticator_to_the_nth_(AuthenticatorID, Authenticators, N) of - {ok, NAuthenticators} -> - NChain = Chain#chain{authenticators = NAuthenticators}, - mnesia:write(?CHAIN_TAB, NChain, write); + gen_server:call(?MODULE, {move_authenticator, ChainID, AuthenticatorID, N}). + +import_users(ChainID, AuthenticatorID, Filename) -> + gen_server:call(?MODULE, {import_users, ChainID, AuthenticatorID, Filename}). + +add_user(ChainID, AuthenticatorID, UserInfo) -> + gen_server:call(?MODULE, {add_user, ChainID, AuthenticatorID, UserInfo}). + +delete_user(ChainID, AuthenticatorID, UserID) -> + gen_server:call(?MODULE, {delete_user, ChainID, AuthenticatorID, UserID}). + +update_user(ChainID, AuthenticatorID, UserID, NewUserInfo) -> + gen_server:call(?MODULE, {update_user, ChainID, AuthenticatorID, UserID, NewUserInfo}). + +lookup_user(ChainID, AuthenticatorID, UserID) -> + gen_server:call(?MODULE, {lookup_user, ChainID, AuthenticatorID, UserID}). + +%% TODO: Support pagination +list_users(ChainID, AuthenticatorID) -> + gen_server:call(?MODULE, {list_users, ChainID, AuthenticatorID}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init(_Opts) -> + ets:new(?CHAIN_TAB, [ named_table, set, public + , {keypos, #chain.id} + , {read_concurrency, true}]), + {ok, #{}}. + +handle_call({create_chain, ID}, _From, State) -> + case ets:member(?CHAIN_TAB, ID) of + true -> + reply({error, {already_exists, {chain, ID}}}, State); + false -> + Chain = #chain{id = ID, + authenticators = [], + created_at = erlang:system_time(millisecond)}, + true = ets:insert(?CHAIN_TAB, Chain), + reply({ok, serialize_chain(Chain)}, State) + end; + +handle_call({delete_chain, ID}, _From, State) -> + case ets:lookup(?CHAIN_TAB, ID) of + [] -> + reply({error, {not_found, {chain, ID}}}, State); + [#chain{authenticators = Authenticators}] -> + _ = [do_delete_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators], + true = ets:delete(?CHAIN_TAB, ID), + reply(ok, State) + end; + +handle_call({lookup_chain, ID}, _From, State) -> + case ets:lookup(?CHAIN_TAB, ID) of + [] -> + reply({error, {not_found, {chain, ID}}}, State); + [Chain] -> + reply({ok, serialize_chain(Chain)}, State) + end; + +handle_call({create_authenticator, ChainID, #{name := Name} = Config}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case lists:keymember(Name, 2, Authenticators) of + true -> + {error, name_has_be_used}; + false -> + AlreadyExist = fun(ID) -> + lists:keymember(ID, 1, Authenticators) + end, + AuthenticatorID = gen_id(AlreadyExist), + case do_create_authenticator(ChainID, AuthenticatorID, Config) of + {ok, Authenticator} -> + NAuthenticators = Authenticators ++ [{AuthenticatorID, Name, Authenticator}], + true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), + {ok, serialize_authenticator(Authenticator)}; {error, Reason} -> {error, Reason} end - end, - update_chain(ChainID, UpdateFun). + end + end, + Reply = update_chain(ChainID, UpdateFun), + reply(Reply, State); -import_users(ChainID, AuthenticatorID, Filename) -> - call_authenticator(ChainID, AuthenticatorID, import_users, [Filename]). +handle_call({delete_authenticator, ChainID, AuthenticatorID}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case lists:keytake(AuthenticatorID, 1, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + {value, {_, _, Authenticator}, NAuthenticators} -> + _ = do_delete_authenticator(Authenticator), + true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), + ok + end + end, + Reply = update_chain(ChainID, UpdateFun), + reply(Reply, State); -add_user(ChainID, AuthenticatorID, UserInfo) -> - call_authenticator(ChainID, AuthenticatorID, add_user, [UserInfo]). +handle_call({update_authenticator, ChainID, AuthenticatorID, Config}, _From, State) -> + Reply = update_or_create_authenticator(ChainID, AuthenticatorID, Config, false), + reply(Reply, State); -delete_user(ChainID, AuthenticatorID, UserID) -> - call_authenticator(ChainID, AuthenticatorID, delete_user, [UserID]). +handle_call({update_or_create_authenticator, ChainID, AuthenticatorID, Config}, _From, State) -> + Reply = update_or_create_authenticator(ChainID, AuthenticatorID, Config, true), + reply(Reply, State); -update_user(ChainID, AuthenticatorID, UserID, NewUserInfo) -> - call_authenticator(ChainID, AuthenticatorID, update_user, [UserID, NewUserInfo]). +handle_call({move_authenticator, ChainID, AuthenticatorID, N}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case move_authenticator_to_the_nth_(AuthenticatorID, Authenticators, N) of + {ok, NAuthenticators} -> + true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), + ok; + {error, Reason} -> + {error, Reason} + end + end, + Reply = update_chain(ChainID, UpdateFun), + reply(Reply, State); -lookup_user(ChainID, AuthenticatorID, UserID) -> - call_authenticator(ChainID, AuthenticatorID, lookup_user, [UserID]). +handle_call({import_users, ChainID, AuthenticatorID, Filename}, _From, State) -> + Reply = call_authenticator(ChainID, AuthenticatorID, import_users, [Filename]), + reply(Reply, State); -list_users(ChainID, AuthenticatorID) -> - call_authenticator(ChainID, AuthenticatorID, list_users, []). +handle_call({add_user, ChainID, AuthenticatorID, UserInfo}, _From, State) -> + Reply = call_authenticator(ChainID, AuthenticatorID, add_user, [UserInfo]), + reply(Reply, State); + +handle_call({delete_user, ChainID, AuthenticatorID, UserID}, _From, State) -> + Reply = call_authenticator(ChainID, AuthenticatorID, delete_user, [UserID]), + reply(Reply, State); + +handle_call({update_user, ChainID, AuthenticatorID, UserID, NewUserInfo}, _From, State) -> + Reply = call_authenticator(ChainID, AuthenticatorID, update_user, [UserID, NewUserInfo]), + reply(Reply, State); + +handle_call({lookup_user, ChainID, AuthenticatorID, UserID}, _From, State) -> + Reply = call_authenticator(ChainID, AuthenticatorID, lookup_user, [UserID]), + reply(Reply, State); + +handle_call({list_users, ChainID, AuthenticatorID}, _From, State) -> + Reply = call_authenticator(ChainID, AuthenticatorID, list_users, []), + reply(Reply, State); + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Req, State) -> + ?LOG(error, "Unexpected case: ~p", [Req]), + {noreply, State}. + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +reply(Reply, State) -> + {reply, Reply, State}. %%------------------------------------------------------------------------------ %% Internal functions @@ -464,6 +476,72 @@ do_create_authenticator(ChainID, AuthenticatorID, #{name := Name} = Config) -> do_delete_authenticator(#authenticator{provider = Provider, state = State}) -> _ = Provider:destroy(State), ok. + +update_or_create_authenticator(ChainID, AuthenticatorID, #{name := NewName} = Config, CreateWhenNotFound) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case lists:keytake(AuthenticatorID, 1, Authenticators) of + false -> + case CreateWhenNotFound of + true -> + case lists:keymember(NewName, 2, Authenticators) of + true -> + {error, name_has_be_used}; + false -> + case do_create_authenticator(ChainID, AuthenticatorID, Config) of + {ok, Authenticator} -> + NAuthenticators = Authenticators ++ [{AuthenticatorID, NewName, Authenticator}], + true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), + {ok, serialize_authenticator(Authenticator)}; + {error, Reason} -> + {error, Reason} + end + end; + false -> + {error, {not_found, {authenticator, AuthenticatorID}}} + end; + {value, + {_, _, #authenticator{provider = Provider, + state = #{version := Version} = State} = Authenticator}, + Others} -> + case lists:keymember(NewName, 2, Others) of + true -> + {error, name_has_be_used}; + false -> + case (NewProvider = authenticator_provider(Config)) =:= Provider of + true -> + Unique = <>, + case Provider:update(Config#{'_unique' => Unique}, State) of + {ok, NewState} -> + NewAuthenticator = Authenticator#authenticator{name = NewName, + config = Config, + state = switch_version(NewState)}, + NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), + true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}), + {ok, serialize_authenticator(NewAuthenticator)}; + {error, Reason} -> + {error, Reason} + end; + false -> + Unique = <>, + case NewProvider:create(Config#{'_unique' => Unique}) of + {ok, NewState} -> + NewAuthenticator = Authenticator#authenticator{name = NewName, + provider = NewProvider, + config = Config, + state = switch_version(NewState)}, + NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), + true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}), + _ = Provider:destroy(State), + {ok, serialize_authenticator(NewAuthenticator)}; + {error, Reason} -> + {error, Reason} + end + end + end + end + end, + update_chain(ChainID, UpdateFun). replace_authenticator(ID, #authenticator{name = Name} = Authenticator, Authenticators) -> lists:keyreplace(ID, 1, Authenticators, {ID, Name, Authenticator}). @@ -487,21 +565,16 @@ move_authenticator_to_the_nth_(AuthenticatorID, [Authenticator | More], N, Passe move_authenticator_to_the_nth_(AuthenticatorID, More, N, [Authenticator | Passed]). update_chain(ChainID, UpdateFun) -> - trans( - fun() -> - case mnesia:read(?CHAIN_TAB, ChainID, write) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [Chain] -> - UpdateFun(Chain) - end - end). - -call_authenticator(ChainID, AuthenticatorID, Func, Args) -> - case mnesia:dirty_read(?CHAIN_TAB, ChainID) of + case ets:lookup(?CHAIN_TAB, ChainID) of [] -> {error, {not_found, {chain, ChainID}}}; - [#chain{authenticators = Authenticators}] -> + [Chain] -> + UpdateFun(Chain) + end. + +call_authenticator(ChainID, AuthenticatorID, Func, Args) -> + UpdateFun = + fun(#chain{authenticators = Authenticators}) -> case lists:keyfind(AuthenticatorID, 1, Authenticators) of false -> {error, {not_found, {authenticator, AuthenticatorID}}}; @@ -513,7 +586,8 @@ call_authenticator(ChainID, AuthenticatorID, Func, Args) -> {error, unsupported_feature} end end - end. + end, + update_chain(ChainID, UpdateFun). serialize_chain(#chain{id = ID, authenticators = Authenticators, @@ -528,12 +602,3 @@ serialize_authenticators(Authenticators) -> serialize_authenticator(#authenticator{id = ID, config = Config}) -> Config#{id => ID}. - -trans(Fun) -> - trans(Fun, []). - -trans(Fun, Args) -> - case ekka_mnesia:transaction(?AUTH_SHARD, Fun, Args) of - {atomic, Res} -> Res; - {aborted, Reason} -> {error, Reason} - end. diff --git a/apps/emqx_authn/src/emqx_authn_sup.erl b/apps/emqx_authn/src/emqx_authn_sup.erl index bb26af0ad..56fcf299a 100644 --- a/apps/emqx_authn/src/emqx_authn_sup.erl +++ b/apps/emqx_authn/src/emqx_authn_sup.erl @@ -26,4 +26,11 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - {ok, {{one_for_one, 10, 10}, []}}. + ChildSpecs = [ + #{id => emqx_authn, + start => {emqx_authn, start_link, []}, + restart => permanent, + type => worker, + modules => [emqx_authn]} + ], + {ok, {{one_for_one, 10, 10}, ChildSpecs}}. From b7bc8b8cacf1cbced204af014cc72eb3ca4ba642 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 12 Aug 2021 13:34:45 +0800 Subject: [PATCH 11/20] feat(authn): improve apis of moving authenticators --- apps/emqx_authn/src/emqx_authn.erl | 113 +++++++++++++----- apps/emqx_authn/src/emqx_authn_api.erl | 62 ++++++---- .../emqx_authn_implied_schema.erl | 9 +- apps/emqx_authn/test/emqx_authn_SUITE.erl | 14 ++- .../test/emqx_authn_mnesia_SUITE.erl | 2 +- 5 files changed, 136 insertions(+), 64 deletions(-) rename apps/emqx_authn/src/{simple_authn => }/emqx_authn_implied_schema.erl (83%) diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 703b0efcf..90114c269 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -49,7 +49,7 @@ , update_or_create_authenticator/3 , lookup_authenticator/2 , list_authenticators/1 - , move_authenticator_to_the_nth/3 + , move_authenticator/3 ]). -export([ import_users/3 @@ -108,6 +108,30 @@ pre_config_update({update_or_create_authenticator, ID, Config}, OldConfig) -> false -> C end end, OldConfig) + end; +pre_config_update({move, ID, Position}, OldConfig) -> + case lookup_authenticator(?CHAIN, ID) of + {error, Reason} -> error(Reason); + {ok, #{name := Name}} -> + {ok, Found, Part1, Part2} = split_by_name(Name, OldConfig), + case Position of + <<"top">> -> + [Found | Part1] ++ Part2; + <<"bottom">> -> + Part1 ++ Part2 ++ [Found]; + Before -> + case binary:split(Before, <<":">>, [global]) of + [<<"before">>, ID0] -> + case lookup_authenticator(?CHAIN, ID0) of + {error, Reason} -> error(Reason); + {ok, #{name := Name1}} -> + {ok, NFound, NPart1, NPart2} = split_by_name(Name1, Part1 + Part2), + NPart1 ++ [Found, NFound | NPart2] + end; + _ -> + error({invalid_parameter, position}) + end + end end. post_config_update({enable, true}, _NewConfig, _OldConfig) -> @@ -157,6 +181,22 @@ post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}}, end; [_Config | _] -> error(name_has_be_used) + end; +post_config_update({move, ID, Position}, _NewConfig, _OldConfig) -> + NPosition = case Position of + <<"top">> -> top; + <<"bottom">> -> bottom; + Before -> + case binary:split(Before, <<":">>, [global]) of + [<<"before">>, ID0] -> + {before, ID0}; + _ -> + error({invalid_parameter, position}) + end + end, + case move_authenticator(?CHAIN, ID, NPosition) of + ok -> ok; + {error, Reason} -> throw(Reason) end. update_config(Path, ConfigRequest) -> @@ -255,8 +295,8 @@ list_authenticators(ChainID) -> {ok, serialize_authenticators(Authenticators)} end. -move_authenticator_to_the_nth(ChainID, AuthenticatorID, N) -> - gen_server:call(?MODULE, {move_authenticator, ChainID, AuthenticatorID, N}). +move_authenticator(ChainID, AuthenticatorID, Position) -> + gen_server:call(?MODULE, {move_authenticator, ChainID, AuthenticatorID, Position}). import_users(ChainID, AuthenticatorID, Filename) -> gen_server:call(?MODULE, {import_users, ChainID, AuthenticatorID, Filename}). @@ -364,16 +404,16 @@ handle_call({update_or_create_authenticator, ChainID, AuthenticatorID, Config}, Reply = update_or_create_authenticator(ChainID, AuthenticatorID, Config, true), reply(Reply, State); -handle_call({move_authenticator, ChainID, AuthenticatorID, N}, _From, State) -> +handle_call({move_authenticator, ChainID, AuthenticatorID, Position}, _From, State) -> UpdateFun = fun(#chain{authenticators = Authenticators} = Chain) -> - case move_authenticator_to_the_nth_(AuthenticatorID, Authenticators, N) of - {ok, NAuthenticators} -> - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - ok; - {error, Reason} -> - {error, Reason} - end + case do_move_authenticator(AuthenticatorID, Authenticators, Position) of + {ok, NAuthenticators} -> + true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), + ok; + {error, Reason} -> + {error, Reason} + end end, Reply = update_chain(ChainID, UpdateFun), reply(Reply, State); @@ -458,6 +498,21 @@ switch_version(State = #{version := ?VER_2}) -> switch_version(State) -> State#{version => ?VER_1}. +split_by_name(Name, Config) -> + {Part1, Part2, true} = lists:foldl( + fun(#{<<"name">> := N} = C, {P1, P2, F0}) -> + F = case N =:= Name of + true -> true; + false -> F0 + end, + case F of + false -> {[C | P1], P2, F}; + true -> {P1, [C | P2], F} + end + end, {[], [], false}, Config), + [Found | NPart2] = lists:reverse(Part2), + {ok, Found, lists:reverse(Part1), NPart2}. + do_create_authenticator(ChainID, AuthenticatorID, #{name := Name} = Config) -> Provider = authenticator_provider(Config), Unique = <>, @@ -546,23 +601,27 @@ update_or_create_authenticator(ChainID, AuthenticatorID, #{name := NewName} = Co replace_authenticator(ID, #authenticator{name = Name} = Authenticator, Authenticators) -> lists:keyreplace(ID, 1, Authenticators, {ID, Name, Authenticator}). -move_authenticator_to_the_nth_(AuthenticatorID, Authenticators, N) - when N =< length(Authenticators) andalso N > 0 -> - move_authenticator_to_the_nth_(AuthenticatorID, Authenticators, N, []); -move_authenticator_to_the_nth_(_, _, _) -> - {error, out_of_range}. +do_move_authenticator(AuthenticatorID, Authenticators, Position) when is_binary(AuthenticatorID) -> + case lists:keytake(AuthenticatorID, 1, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + {value, Authenticator, NAuthenticators} -> + do_move_authenticator(Authenticator, NAuthenticators, Position) + end; -move_authenticator_to_the_nth_(AuthenticatorID, [], _, _) -> - {error, {not_found, {authenticator, AuthenticatorID}}}; -move_authenticator_to_the_nth_(AuthenticatorID, [{AuthenticatorID, _, _} = Authenticator | More], N, Passed) - when N =< length(Passed) -> - {L1, L2} = lists:split(N - 1, lists:reverse(Passed)), - {ok, L1 ++ [Authenticator] ++ L2 ++ More}; -move_authenticator_to_the_nth_(AuthenticatorID, [{AuthenticatorID, _, _} = Authenticator | More], N, Passed) -> - {L1, L2} = lists:split(N - length(Passed) - 1, More), - {ok, lists:reverse(Passed) ++ L1 ++ [Authenticator] ++ L2}; -move_authenticator_to_the_nth_(AuthenticatorID, [Authenticator | More], N, Passed) -> - move_authenticator_to_the_nth_(AuthenticatorID, More, N, [Authenticator | Passed]). +do_move_authenticator(Authenticator, Authenticators, top) -> + {ok, [Authenticator | Authenticators]}; +do_move_authenticator(Authenticator, Authenticators, bottom) -> + {ok, Authenticators ++ [Authenticator]}; +do_move_authenticator(Authenticator, Authenticators, {before, ID}) -> + insert(Authenticator, Authenticators, ID, []). + +insert(_, [], ID, _) -> + {error, {not_found, {authenticator, ID}}}; +insert(Authenticator, [{ID, _, _} | _] = Authenticators, ID, Acc) -> + {ok, lists:reverse(Acc) ++ [Authenticator | Authenticators]}; +insert(Authenticator, [{_, _, _} = Authenticator0 | More], ID, Acc) -> + insert(Authenticator, More, ID, [Authenticator0 | Acc]). update_chain(ChainID, UpdateFun) -> case ets:lookup(?CHAIN_TAB, ChainID) of diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 1e232d553..2503adf10 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -24,7 +24,7 @@ , authentication/2 , authenticators/2 , authenticators2/2 - , position/2 + , move/2 , import_users/2 , users/2 , users2/2 @@ -109,7 +109,7 @@ api_spec() -> {[ authentication_api() , authenticators_api() , authenticators_api2() - , position_api() + , move_api() , import_users_api() , users_api() , users2_api() @@ -405,10 +405,10 @@ authenticators_api2() -> }, {"/authentication/authenticators/:id", Metadata, authenticators2}. -position_api() -> +move_api() -> Metadata = #{ post => #{ - description => "Change the order of authenticators", + description => "Move authenticator", parameters => [ #{ name => id, @@ -423,14 +423,30 @@ position_api() -> content => #{ 'application/json' => #{ schema => #{ - type => object, - required => [position], - properties => #{ - position => #{ - type => integer, - example => 1 + oneOf => [ + #{ + type => object, + required => [position], + properties => #{ + position => #{ + type => string, + enum => [<<"top">>, <<"bottom">>], + example => <<"top">> + } + } + }, + #{ + type => object, + required => [position], + properties => #{ + position => #{ + type => string, + description => <<"before:">>, + example => <<"before:67e4c9d3">> + } + } } - } + ] } } } @@ -444,7 +460,7 @@ position_api() -> } } }, - {"/authentication/authenticators/:id/position", Metadata, position}. + {"/authentication/authenticators/:id/move", Metadata, move}. import_users_api() -> Metadata = #{ @@ -1304,18 +1320,17 @@ authenticators2(delete, Request) -> serialize_error(Reason) end. -position(post, Request) -> +move(post, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), {ok, Body, _} = cowboy_req:read_body(Request), - NBody = emqx_json:decode(Body, [return_maps]), - Config = hocon_schema:check_plain(emqx_authn_implied_schema, #{<<"position">> => NBody}, - #{nullable => true}, ["position"]), - #{position := #{position := Position}} = emqx_map_lib:unsafe_atom_key_map(Config), - case emqx_authn:move_authenticator_to_the_nth(?CHAIN, AuthenticatorID, Position) of - ok -> - {204}; - {error, Reason} -> - serialize_error(Reason) + case emqx_json:decode(Body, [return_maps]) of + #{<<"position">> := Position} -> + case emqx_authn:update_config([authentication, authenticators], {move_authenticator, AuthenticatorID, Position}) of + ok -> {204}; + {error, Reason} -> serialize_error(Reason) + end; + _ -> + serialize_error({missing_parameter, position}) end. import_users(post, Request) -> @@ -1393,9 +1408,6 @@ serialize_error({not_found, {authenticator, ID}}) -> serialize_error(name_has_be_used) -> {409, #{code => <<"ALREADY_EXISTS">>, message => <<"Name has be used">>}}; -serialize_error(out_of_range) -> - {400, #{code => <<"OUT_OF_RANGE">>, - message => <<"Out of range">>}}; serialize_error({missing_parameter, Name}) -> {400, #{code => <<"MISSING_PARAMETER">>, message => list_to_binary( diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_implied_schema.erl b/apps/emqx_authn/src/emqx_authn_implied_schema.erl similarity index 83% rename from apps/emqx_authn/src/simple_authn/emqx_authn_implied_schema.erl rename to apps/emqx_authn/src/emqx_authn_implied_schema.erl index 1a0731e92..65b4bf356 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_implied_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_implied_schema.erl @@ -25,12 +25,10 @@ , fields/1 ]). -structs() -> [ "filename", "position", "user_info", "new_user_info"]. +structs() -> [ "filename", "user_info", "new_user_info"]. fields("filename") -> [ {filename, fun filename/1} ]; -fields("position") -> - [ {position, fun position/1} ]; fields("user_info") -> [ {user_id, fun user_id/1} , {password, fun password/1} @@ -43,11 +41,6 @@ filename(type) -> string(); filename(nullable) -> false; filename(_) -> undefined. -position(type) -> integer(); -position(validate) -> [fun (Position) -> Position > 0 end]; -position(nullable) -> false; -position(_) -> undefined. - user_id(type) -> binary(); user_id(nullable) -> false; user_id(_) -> undefined. diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index 92e506d51..9c4371838 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -86,10 +86,18 @@ t_authenticator(_) -> ?assertMatch({ok, #{id := ?CHAIN, authenticators := [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}}, ?AUTH:lookup_chain(?CHAIN)), ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), - ?assertEqual(ok, ?AUTH:move_authenticator_to_the_nth(?CHAIN, ID2, 1)), + ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), ?assertMatch({ok, [#{name := AuthenticatorName2}, #{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), - ?assertEqual({error, out_of_range}, ?AUTH:move_authenticator_to_the_nth(?CHAIN, ID2, 3)), - ?assertEqual({error, out_of_range}, ?AUTH:move_authenticator_to_the_nth(?CHAIN, ID2, 0)), + + ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, bottom)), + ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), + + ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, {before, ID1})), + + ?assertMatch({ok, [#{name := AuthenticatorName2}, #{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), + + ?assertEqual({error, {not_found, {authenticator, <<"nonexistent">>}}}, ?AUTH:move_authenticator(?CHAIN, ID2, {before, <<"nonexistent">>})), + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), ?assertEqual({ok, []}, ?AUTH:list_authenticators(?CHAIN)), diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index 4a5a24844..fdcaf519d 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -144,7 +144,7 @@ t_multi_mnesia_authenticator(_) -> clientid => <<"myclient">>, password => <<"mypass1">>}, ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo1, ok)), - ?assertEqual(ok, ?AUTH:move_authenticator_to_the_nth(?CHAIN, ID2, 1)), + ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo1, ok)), ClientInfo2 = ClientInfo1#{password => <<"mypass2">>}, From 3f2ca5282c3db97357b3bd1abe9b20397bad22bb Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 12 Aug 2021 13:48:25 +0800 Subject: [PATCH 12/20] chore(authn): remove implied schema --- apps/emqx_authn/src/emqx_authn_api.erl | 61 +++++++++++-------- .../src/emqx_authn_implied_schema.erl | 51 ---------------- 2 files changed, 34 insertions(+), 78 deletions(-) delete mode 100644 apps/emqx_authn/src/emqx_authn_implied_schema.erl diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 2503adf10..c4a12309e 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1336,29 +1336,35 @@ move(post, Request) -> import_users(post, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), {ok, Body, _} = cowboy_req:read_body(Request), - NBody = emqx_json:decode(Body, [return_maps]), - Config = hocon_schema:check_plain(emqx_authn_implied_schema, #{<<"filename">> => NBody}, - #{nullable => true}, ["filename"]), - #{filename := #{filename := Filename}} = emqx_map_lib:unsafe_atom_key_map(Config), - case emqx_authn:import_users(?CHAIN, AuthenticatorID, Filename) of - ok -> - {204}; - {error, Reason} -> - serialize_error(Reason) + case emqx_json:decode(Body, [return_maps]) of + #{<<"filename">> := Filename} -> + case emqx_authn:import_users(?CHAIN, AuthenticatorID, Filename) of + ok -> + {204}; + {error, Reason} -> + serialize_error(Reason) + end; + _ -> + serialize_error({missing_parameter, filename}) end. users(post, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), {ok, Body, _} = cowboy_req:read_body(Request), - NBody = emqx_json:decode(Body, [return_maps]), - Config = hocon_schema:check_plain(emqx_authn_implied_schema, #{<<"user_info">> => NBody}, - #{nullable => true}, ["user_info"]), - #{user_info := UserInfo} = emqx_map_lib:unsafe_atom_key_map(Config), - case emqx_authn:add_user(?CHAIN, AuthenticatorID, UserInfo) of - {ok, User} -> - {201, User}; - {error, Reason} -> - serialize_error(Reason) + case emqx_json:decode(Body, [return_maps]) of + #{ <<"user_id">> := UserID + , <<"password">> := Password} -> + case emqx_authn:add_user(?CHAIN, AuthenticatorID, #{ user_id => UserID + , password => Password}) of + {ok, User} -> + {201, User}; + {error, Reason} -> + serialize_error(Reason) + end; + #{<<"user_id">> := _} -> + serialize_error({missing_parameter, password}); + _ -> + serialize_error({missing_parameter, user_id}) end; users(get, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), @@ -1373,15 +1379,16 @@ users2(patch, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), UserID = cowboy_req:binding(user_id, Request), {ok, Body, _} = cowboy_req:read_body(Request), - NBody = emqx_json:decode(Body, [return_maps]), - Config = hocon_schema:check_plain(emqx_authn_implied_schema, #{<<"new_user_info">> => NBody}, - #{nullable => true}, ["new_user_info"]), - #{new_user_info := NewUserInfo} = emqx_map_lib:unsafe_atom_key_map(Config), - case emqx_authn:update_user(?CHAIN, AuthenticatorID, UserID, NewUserInfo) of - {ok, User} -> - {200, User}; - {error, Reason} -> - serialize_error(Reason) + case emqx_json:decode(Body, [return_maps]) of + #{<<"password">> := Password} -> + case emqx_authn:update_user(?CHAIN, AuthenticatorID, UserID, #{password => Password}) of + {ok, User} -> + {200, User}; + {error, Reason} -> + serialize_error(Reason) + end; + _ -> + serialize_error({missing_parameter, password}) end; users2(get, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), diff --git a/apps/emqx_authn/src/emqx_authn_implied_schema.erl b/apps/emqx_authn/src/emqx_authn_implied_schema.erl deleted file mode 100644 index 65b4bf356..000000000 --- a/apps/emqx_authn/src/emqx_authn_implied_schema.erl +++ /dev/null @@ -1,51 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authn_implied_schema). - --include("emqx_authn.hrl"). --include_lib("typerefl/include/types.hrl"). - --behaviour(hocon_schema). - --export([ structs/0 - , fields/1 - ]). - -structs() -> [ "filename", "user_info", "new_user_info"]. - -fields("filename") -> - [ {filename, fun filename/1} ]; -fields("user_info") -> - [ {user_id, fun user_id/1} - , {password, fun password/1} - ]; -fields("new_user_info") -> - [ {password, fun password/1} - ]. - -filename(type) -> string(); -filename(nullable) -> false; -filename(_) -> undefined. - -user_id(type) -> binary(); -user_id(nullable) -> false; -user_id(_) -> undefined. - -password(type) -> binary(); -password(nullable) -> false; -password(_) -> undefined. - From 429def6b95c877afec4685036a8216592b6514b4 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 12 Aug 2021 13:55:21 +0800 Subject: [PATCH 13/20] fix(authn): fix http api spec --- apps/emqx_authn/src/emqx_authn_api.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index c4a12309e..bcaaebe93 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -1070,7 +1070,13 @@ definitions() -> PasswordBasedRedisDef = #{ type => object, - required => [], + required => [ server_type + , server + , servers + , password + , database + , query + ], properties => #{ server_type => #{ type => string, @@ -1099,7 +1105,7 @@ definitions() -> }, database => #{ type => integer, - exmaple => 0 + example => 0 }, query => #{ type => string, From e6f9767066ebc651ea3de40543fd943b541971a3 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Fri, 13 Aug 2021 10:32:31 +0800 Subject: [PATCH 14/20] feat(authn): support superuser --- apps/emqx/src/emqx_access_control.erl | 9 +- apps/emqx/src/emqx_channel.erl | 19 ++-- apps/emqx_authn/data/user-credentials.csv | 6 +- apps/emqx_authn/data/user-credentials.json | 6 +- apps/emqx_authn/src/emqx_authn.erl | 5 +- apps/emqx_authn/src/emqx_authn_api.erl | 42 ++++++--- .../emqx_enhanced_authn_scram_mnesia.erl | 88 +++++++++++++------ .../src/simple_authn/emqx_authn_http.erl | 13 +-- .../src/simple_authn/emqx_authn_jwt.erl | 9 +- .../src/simple_authn/emqx_authn_mnesia.erl | 75 +++++++++------- .../src/simple_authn/emqx_authn_mongodb.erl | 8 +- .../src/simple_authn/emqx_authn_mysql.erl | 18 ++-- .../src/simple_authn/emqx_authn_pgsql.erl | 22 +++-- .../src/simple_authn/emqx_authn_redis.erl | 12 ++- .../emqx_authn/test/data/user-credentials.csv | 6 +- .../test/data/user-credentials.json | 6 +- apps/emqx_authn/test/emqx_authn_SUITE.erl | 4 +- apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl | 30 ++++--- .../test/emqx_authn_mnesia_SUITE.erl | 43 +++++---- 19 files changed, 270 insertions(+), 151 deletions(-) diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 65991d222..111a86112 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -27,9 +27,14 @@ %%-------------------------------------------------------------------- -spec(authenticate(emqx_types:clientinfo()) -> - ok | {ok, binary()} | {continue, map()} | {continue, binary(), map()} | {error, term()}). + {ok, map()} | {ok, map(), binary()} | {continue, map()} | {continue, binary(), map()} | {error, term()}). authenticate(Credential) -> - run_hooks('client.authenticate', [Credential], ok). + case run_hooks('client.authenticate', [Credential], {ok, #{superuser => false}}) of + ok -> + {ok, #{superuser => false}}; + Other -> + Other + end. %% @doc Check Authorization -spec authorize(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic()) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 5e4d11953..93d5e2c37 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1299,14 +1299,17 @@ authenticate(?AUTH_PACKET(_, #{'Authentication-Method' := AuthMethod} = Properti {error, ?RC_BAD_AUTHENTICATION_METHOD} end. -do_authenticate(#{auth_method := AuthMethod} = Credential, Channel) -> +do_authenticate(#{auth_method := AuthMethod} = Credential, #channel{clientinfo = ClientInfo} = Channel) -> Properties = #{'Authentication-Method' => AuthMethod}, case emqx_access_control:authenticate(Credential) of - ok -> - {ok, Properties, Channel#channel{auth_cache = #{}}}; - {ok, AuthData} -> + {ok, #{superuser := Superuser}} -> + {ok, Properties, + Channel#channel{clientinfo = ClientInfo#{is_superuser => Superuser}, + auth_cache = #{}}}; + {ok, #{superuser := Superuser}, AuthData} -> {ok, Properties#{'Authentication-Data' => AuthData}, - Channel#channel{auth_cache = #{}}}; + Channel#channel{clientinfo = ClientInfo#{is_superuser => Superuser}, + auth_cache = #{}}}; {continue, AuthCache} -> {continue, Properties, Channel#channel{auth_cache = AuthCache}}; {continue, AuthData, AuthCache} -> @@ -1316,10 +1319,10 @@ do_authenticate(#{auth_method := AuthMethod} = Credential, Channel) -> {error, emqx_reason_codes:connack_error(Reason)} end; -do_authenticate(Credential, Channel) -> +do_authenticate(Credential, #channel{clientinfo = ClientInfo} = Channel) -> case emqx_access_control:authenticate(Credential) of - ok -> - {ok, #{}, Channel}; + {ok, #{superuser := Superuser}} -> + {ok, #{}, Channel#channel{clientinfo = ClientInfo#{is_superuser => Superuser}}}; {error, Reason} -> {error, emqx_reason_codes:connack_error(Reason)} end. diff --git a/apps/emqx_authn/data/user-credentials.csv b/apps/emqx_authn/data/user-credentials.csv index 2543d39ca..0548308b7 100644 --- a/apps/emqx_authn/data/user-credentials.csv +++ b/apps/emqx_authn/data/user-credentials.csv @@ -1,3 +1,3 @@ -user_id,password_hash,salt -myuser3,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235 -myuser4,ee68c985a69208b6eda8c6c9b4c7c2d2b15ee2352cdd64a903171710a99182e8,ad773b5be9dd0613fe6c2f4d8c403139 +user_id,password_hash,salt,superuser +myuser3,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true +myuser4,ee68c985a69208b6eda8c6c9b4c7c2d2b15ee2352cdd64a903171710a99182e8,ad773b5be9dd0613fe6c2f4d8c403139,false diff --git a/apps/emqx_authn/data/user-credentials.json b/apps/emqx_authn/data/user-credentials.json index 169122bd2..e54501233 100644 --- a/apps/emqx_authn/data/user-credentials.json +++ b/apps/emqx_authn/data/user-credentials.json @@ -2,11 +2,13 @@ { "user_id":"myuser1", "password_hash":"c5e46903df45e5dc096dc74657610dbee8deaacae656df88a1788f1847390242", - "salt": "e378187547bf2d6f0545a3f441aa4d8a" + "salt": "e378187547bf2d6f0545a3f441aa4d8a", + "superuser": true }, { "user_id":"myuser2", "password_hash":"f4d17f300b11e522fd33f497c11b126ef1ea5149c74d2220f9a16dc876d4567b", - "salt": "6d3f9bd5b54d94b98adbcfe10b6d181f" + "salt": "6d3f9bd5b54d94b98adbcfe10b6d181f", + "superuser": false } ] diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 90114c269..384830750 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -235,8 +235,9 @@ do_authenticate([{_, _, #authenticator{provider = Provider, state = State}} | Mo ignore -> do_authenticate(More, Credential); Result -> - %% ok - %% {ok, AuthData} + %% {ok, Extra} + %% {ok, Extra, AuthData} + %% {ok, MetaData} %% {continue, AuthCache} %% {continue, AuthData, AuthCache} %% {error, Reason} diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index bcaaebe93..97ed94a8b 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -528,6 +528,10 @@ users_api() -> }, password => #{ type => string + }, + superuser => #{ + type => boolean, + default => false } } } @@ -541,10 +545,12 @@ users_api() -> 'application/json' => #{ schema => #{ type => object, - required => [user_id], properties => #{ user_id => #{ type => string + }, + superuser => #{ + type => boolean } } } @@ -576,10 +582,12 @@ users_api() -> type => array, items => #{ type => object, - required => [user_id], properties => #{ user_id => #{ type => string + }, + superuser => #{ + type => boolean } } } @@ -620,10 +628,12 @@ users2_api() -> 'application/json' => #{ schema => #{ type => object, - required => [password], properties => #{ password => #{ type => string + }, + superuser => #{ + type => boolean } } } @@ -642,6 +652,9 @@ users2_api() -> properties => #{ user_id => #{ type => string + }, + superuser => #{ + type => boolean } } } @@ -685,6 +698,9 @@ users2_api() -> properties => #{ user_id => #{ type => string + }, + superuser => #{ + type => boolean } } } @@ -1359,9 +1375,11 @@ users(post, Request) -> {ok, Body, _} = cowboy_req:read_body(Request), case emqx_json:decode(Body, [return_maps]) of #{ <<"user_id">> := UserID - , <<"password">> := Password} -> + , <<"password">> := Password} = UserInfo -> + Superuser = maps:get(<<"superuser">>, UserInfo, false), case emqx_authn:add_user(?CHAIN, AuthenticatorID, #{ user_id => UserID - , password => Password}) of + , password => Password + , superuser => Superuser}) of {ok, User} -> {201, User}; {error, Reason} -> @@ -1385,16 +1403,18 @@ users2(patch, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), UserID = cowboy_req:binding(user_id, Request), {ok, Body, _} = cowboy_req:read_body(Request), - case emqx_json:decode(Body, [return_maps]) of - #{<<"password">> := Password} -> - case emqx_authn:update_user(?CHAIN, AuthenticatorID, UserID, #{password => Password}) of + UserInfo = emqx_json:decode(Body, [return_maps]), + NUserInfo = maps:with([<<"password">>, <<"superuser">>], UserInfo), + case NUserInfo =:= #{} of + true -> + serialize_error({missing_parameter, password}); + false -> + case emqx_authn:update_user(?CHAIN, AuthenticatorID, UserID, UserInfo) of {ok, User} -> {200, User}; {error, Reason} -> serialize_error(Reason) - end; - _ -> - serialize_error({missing_parameter, password}) + end end; users2(get, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 56629c568..98cdb8c26 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -17,7 +17,6 @@ -module(emqx_enhanced_authn_scram_mnesia). -include("emqx_authn.hrl"). --include_lib("esasl/include/esasl_scram.hrl"). -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). @@ -48,6 +47,14 @@ -rlog_shard({?AUTH_SHARD, ?TAB}). +-record(user_info, + { user_id + , stored_key + , server_key + , salt + , superuser + }). + %%------------------------------------------------------------------------------ %% Mnesia bootstrap %%------------------------------------------------------------------------------ @@ -57,8 +64,8 @@ mnesia(boot) -> ok = ekka_mnesia:create_table(?TAB, [ {disc_copies, [node()]}, - {record_name, scram_user_credentail}, - {attributes, record_info(fields, scram_user_credentail)}, + {record_name, user_info}, + {attributes, record_info(fields, user_info)}, {storage_properties, [{ets, [{read_concurrency, true}]}]}]); mnesia(copy) -> @@ -126,20 +133,21 @@ authenticate(_Credential, _State) -> destroy(#{user_group := UserGroup}) -> trans( fun() -> - MatchSpec = [{{scram_user_credentail, {UserGroup, '_'}, '_', '_', '_'}, [], ['$_']}], - ok = lists:foreach(fun(UserCredential) -> - mnesia:delete_object(?TAB, UserCredential, write) + MatchSpec = [{{user_info, {UserGroup, '_'}, '_', '_', '_', '_'}, [], ['$_']}], + ok = lists:foreach(fun(UserInfo) -> + mnesia:delete_object(?TAB, UserInfo, write) end, mnesia:select(?TAB, MatchSpec, write)) end). add_user(#{user_id := UserID, - password := Password}, #{user_group := UserGroup} = State) -> + password := Password} = UserInfo, #{user_group := UserGroup} = State) -> trans( fun() -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> - add_user(UserID, Password, State), - {ok, #{user_id => UserID}}; + Superuser = maps:get(superuser, UserInfo, false), + add_user(UserID, Password, Superuser, State), + {ok, #{user_id => UserID, superuser => Superuser}}; [_] -> {error, already_exist} end @@ -156,31 +164,41 @@ delete_user(UserID, #{user_group := UserGroup}) -> end end). -update_user(UserID, #{password := Password}, +update_user(UserID, User, #{user_group := UserGroup} = State) -> trans( fun() -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> {error, not_found}; - [_] -> - add_user(UserID, Password, State), - {ok, #{user_id => UserID}} + [#user_info{superuser = Superuser} = UserInfo] -> + UserInfo1 = UserInfo#user_info{superuser = maps:get(superuser, User, Superuser)}, + UserInfo2 = case maps:get(password, User, undefined) of + undefined -> + UserInfo1; + Password -> + {StoredKey, ServerKey, Salt} = esasl_scram:generate_user_credential(Password, State), + UserInfo1#user_info{stored_key = StoredKey, + server_key = ServerKey, + salt = Salt} + end, + mnesia:write(?TAB, UserInfo2, write), + {ok, serialize_user_info(UserInfo2)} end end). lookup_user(UserID, #{user_group := UserGroup}) -> case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of - [#scram_user_credentail{user_id = {_, UserID}}] -> - {ok, #{user_id => UserID}}; + [UserInfo] -> + {ok, serialize_user_info(UserInfo)}; [] -> {error, not_found} end. %% TODO: Support Pagination list_users(#{user_group := UserGroup}) -> - Users = [#{user_id => UserID} || - #scram_user_credentail{user_id = {UserGroup0, UserID}} <- ets:tab2list(?TAB), UserGroup0 =:= UserGroup], + Users = [serialize_user_info(UserInfo) || + #user_info{user_id = {UserGroup0, _}} = UserInfo <- ets:tab2list(?TAB), UserGroup0 =:= UserGroup], {ok, Users}. %%------------------------------------------------------------------------------ @@ -195,13 +213,13 @@ ensure_auth_method(_, _) -> false. check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = State) -> - LookupFun = fun(Username) -> - lookup_user2(Username, State) + RetrieveFun = fun(Username) -> + retrieve(Username, State) end, case esasl_scram:check_client_first_message( Bin, #{iteration_count => IterationCount, - lookup => LookupFun} + retrieve => RetrieveFun} ) of {cotinue, ServerFirstMessage, Cache} -> {cotinue, ServerFirstMessage, Cache}; @@ -209,25 +227,36 @@ check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = S {error, not_authorized} end. -check_client_final_message(Bin, Cache, #{algorithm := Alg}) -> +check_client_final_message(Bin, #{superuser := Superuser} = Cache, #{algorithm := Alg}) -> case esasl_scram:check_client_final_message( Bin, Cache#{algorithm => Alg} ) of {ok, ServerFinalMessage} -> - {ok, ServerFinalMessage}; + {ok, #{superuser => Superuser}, ServerFinalMessage}; {error, _Reason} -> {error, not_authorized} end. -add_user(UserID, Password, State) -> - UserCredential = esasl_scram:generate_user_credential(UserID, Password, State), - mnesia:write(?TAB, UserCredential, write). +add_user(UserID, Password, Superuser, State) -> + {StoredKey, ServerKey, Salt} = esasl_scram:generate_user_credential(Password, State), + UserInfo = #user_info{user_id = UserID, + stored_key = StoredKey, + server_key = ServerKey, + salt = Salt, + superuser = Superuser}, + mnesia:write(?TAB, UserInfo, write). -lookup_user2(UserID, #{user_group := UserGroup}) -> +retrieve(UserID, #{user_group := UserGroup}) -> case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of - [#scram_user_credentail{} = UserCredential] -> - {ok, UserCredential}; + [#user_info{stored_key = StoredKey, + server_key = ServerKey, + salt = Salt, + superuser = Superuser}] -> + {ok, #{stored_key => StoredKey, + server_key => ServerKey, + salt => Salt, + superuser => Superuser}}; [] -> {error, not_found} end. @@ -241,3 +270,6 @@ trans(Fun, Args) -> {atomic, Res} -> Res; {aborted, Reason} -> {error, Reason} end. + +serialize_user_info(#user_info{user_id = {_, UserID}, superuser = Superuser}) -> + #{user_id => UserID, superuser => Superuser}. \ No newline at end of file diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index aa10a3b98..026df2415 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -154,15 +154,16 @@ authenticate(Credential, #{'_unique' := Unique, try Request = generate_request(Credential, State), case emqx_resource:query(Unique, {Method, Request, RequestTimeout}) of - {ok, 204, _Headers} -> ok; + {ok, 204, _Headers} -> {ok, #{superuser => false}}; {ok, 200, Headers, Body} -> ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>), case safely_parse_body(ContentType, Body) of - {ok, _NBody} -> + {ok, NBody} -> %% TODO: Return by user property - ok; + {ok, #{superuser => maps:get(<<"superuser">>, NBody, false), + user_property => NBody}}; {error, _Reason} -> - ok + {ok, #{superuser => false}} end; {error, _Reason} -> ignore @@ -291,8 +292,8 @@ safely_parse_body(ContentType, Body) -> end. parse_body(<<"application/json">>, Body) -> - {ok, emqx_json:decode(Body)}; + {ok, emqx_json:decode(Body, [return_maps])}; parse_body(<<"application/x-www-form-urlencoded">>, Body) -> - {ok, cow_qs:parse_qs(Body)}; + {ok, maps:from_list(cow_qs:parse_qs(Body))}; parse_body(ContentType, _) -> {error, {unsupported_content_type, ContentType}}. \ No newline at end of file diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index fe034994e..74aa9e8f6 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -169,7 +169,7 @@ authenticate(Credential = #{password := JWT}, #{jwk := JWK, end, VerifyClaims = replace_placeholder(VerifyClaims0, Credential), case verify(JWT, JWKs, VerifyClaims) of - ok -> ok; + {ok, Extra} -> {ok, Extra}; {error, invalid_signature} -> ignore; {error, {claims, _}} -> {error, bad_username_or_password} end. @@ -239,7 +239,12 @@ verify(JWS, [JWK | More], VerifyClaims) -> try jose_jws:verify(JWK, JWS) of {true, Payload, _JWS} -> Claims = emqx_json:decode(Payload, [return_maps]), - verify_claims(Claims, VerifyClaims); + case verify_claims(Claims, VerifyClaims) of + ok -> + {ok, #{superuser => maps:get(<<"superuser">>, Claims, false)}}; + {error, Reason} -> + {error, Reason} + end; {false, _, _} -> verify(JWS, More, VerifyClaims) catch diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index ce845d4e3..08c0ffad1 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -46,6 +46,7 @@ { user_id :: {user_group(), user_id()} , password_hash :: binary() , salt :: binary() + , superuser :: boolean() }). -reflect_type([ user_id_type/0 ]). @@ -147,13 +148,13 @@ authenticate(#{password := Password} = Credential, case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of [] -> ignore; - [#user_info{password_hash = PasswordHash, salt = Salt0}] -> + [#user_info{password_hash = PasswordHash, salt = Salt0, superuser = Superuser}] -> Salt = case Algorithm of bcrypt -> PasswordHash; _ -> Salt0 end, case PasswordHash =:= hash(Algorithm, Password, Salt) of - true -> ok; + true -> {ok, #{superuser => Superuser}}; false -> {error, bad_username_or_password} end end. @@ -161,7 +162,7 @@ authenticate(#{password := Password} = Credential, destroy(#{user_group := UserGroup}) -> trans( fun() -> - MatchSpec = [{{user_info, {UserGroup, '_'}, '_', '_'}, [], ['$_']}], + MatchSpec = [{{user_info, {UserGroup, '_'}, '_', '_', '_'}, [], ['$_']}], ok = lists:foreach(fun delete_user2/1, mnesia:select(?TAB, MatchSpec, write)) end). @@ -179,14 +180,16 @@ import_users(Filename0, State) -> end. add_user(#{user_id := UserID, - password := Password}, + password := Password} = UserInfo, #{user_group := UserGroup} = State) -> trans( fun() -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> - add(UserID, Password, State), - {ok, #{user_id => UserID}}; + {PasswordHash, Salt} = hash(Password, State), + Superuser = maps:get(superuser, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), + {ok, #{user_id => UserID, superuser => Superuser}}; [_] -> {error, already_exist} end @@ -203,29 +206,38 @@ delete_user(UserID, #{user_group := UserGroup}) -> end end). -update_user(UserID, #{password := Password}, +update_user(UserID, UserInfo, #{user_group := UserGroup} = State) -> trans( fun() -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> {error, not_found}; - [_] -> - add(UserID, Password, State), - {ok, #{user_id => UserID}} + [#user_info{ password_hash = PasswordHash + , salt = Salt + , superuser = Superuser}] -> + NSuperuser = maps:get(superuser, UserInfo, Superuser), + {NPasswordHash, NSalt} = case maps:get(password, UserInfo, undefined) of + undefined -> + {PasswordHash, Salt}; + Password -> + hash(Password, State) + end, + insert_user(UserGroup, UserID, NPasswordHash, NSalt, NSuperuser), + {ok, #{user_id => UserID, superuser => NSuperuser}} end end). lookup_user(UserID, #{user_group := UserGroup}) -> case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of - [#user_info{user_id = {_, UserID}}] -> - {ok, #{user_id => UserID}}; + [UserInfo] -> + {ok, serialize_user_info(UserInfo)}; [] -> {error, not_found} end. list_users(#{user_group := UserGroup}) -> - Users = [#{user_id => UserID} || #user_info{user_id = {UserGroup0, UserID}} <- ets:tab2list(?TAB), UserGroup0 =:= UserGroup], + Users = [serialize_user_info(UserInfo) || #user_info{user_id = {UserGroup0, _}} = UserInfo <- ets:tab2list(?TAB), UserGroup0 =:= UserGroup], {ok, Users}. %%------------------------------------------------------------------------------ @@ -268,7 +280,8 @@ import(UserGroup, [#{<<"user_id">> := UserID, <<"password_hash">> := PasswordHash} = UserInfo | More]) when is_binary(UserID) andalso is_binary(PasswordHash) -> Salt = maps:get(<<"salt">>, UserInfo, <<>>), - insert_user(UserGroup, UserID, PasswordHash, Salt), + Superuser = maps:get(<<"superuser">>, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), import(UserGroup, More); import(_UserGroup, [_ | _More]) -> {error, bad_format}. @@ -282,7 +295,8 @@ import(UserGroup, File, Seq) -> {ok, #{user_id := UserID, password_hash := PasswordHash} = UserInfo} -> Salt = maps:get(salt, UserInfo, <<>>), - insert_user(UserGroup, UserID, PasswordHash, Salt), + Superuser = maps:get(superuser, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), import(UserGroup, File, Seq); {error, Reason} -> {error, Reason} @@ -307,8 +321,6 @@ get_csv_header(File) -> get_user_info_by_seq(Fields, Seq) -> get_user_info_by_seq(Fields, Seq, #{}). -get_user_info_by_seq([], [], #{user_id := _, password_hash := _, salt := _} = Acc) -> - {ok, Acc}; get_user_info_by_seq([], [], #{user_id := _, password_hash := _} = Acc) -> {ok, Acc}; get_user_info_by_seq(_, [], _) -> @@ -319,19 +331,13 @@ get_user_info_by_seq([PasswordHash | More1], [<<"password_hash">> | More2], Acc) get_user_info_by_seq(More1, More2, Acc#{password_hash => PasswordHash}); get_user_info_by_seq([Salt | More1], [<<"salt">> | More2], Acc) -> get_user_info_by_seq(More1, More2, Acc#{salt => Salt}); +get_user_info_by_seq([<<"true">> | More1], [<<"superuser">> | More2], Acc) -> + get_user_info_by_seq(More1, More2, Acc#{superuser => true}); +get_user_info_by_seq([<<"false">> | More1], [<<"superuser">> | More2], Acc) -> + get_user_info_by_seq(More1, More2, Acc#{superuser => false}); get_user_info_by_seq(_, _, _) -> {error, bad_format}. --compile({inline, [add/3]}). -add(UserID, Password, #{user_group := UserGroup, - password_hash_algorithm := Algorithm} = State) -> - Salt = gen_salt(State), - PasswordHash = hash(Algorithm, Password, Salt), - case Algorithm of - bcrypt -> insert_user(UserGroup, UserID, PasswordHash); - _ -> insert_user(UserGroup, UserID, PasswordHash, Salt) - end. - gen_salt(#{password_hash_algorithm := plain}) -> <<>>; gen_salt(#{password_hash_algorithm := bcrypt, @@ -347,13 +353,16 @@ hash(bcrypt, Password, Salt) -> hash(Algorithm, Password, Salt) -> emqx_passwd:hash(Algorithm, <>). -insert_user(UserGroup, UserID, PasswordHash) -> - insert_user(UserGroup, UserID, PasswordHash, <<>>). +hash(Password, #{password_hash_algorithm := Algorithm} = State) -> + Salt = gen_salt(State), + PasswordHash = hash(Algorithm, Password, Salt), + {PasswordHash, Salt}. -insert_user(UserGroup, UserID, PasswordHash, Salt) -> +insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser) -> UserInfo = #user_info{user_id = {UserGroup, UserID}, password_hash = PasswordHash, - salt = Salt}, + salt = Salt, + superuser = Superuser}, mnesia:write(?TAB, UserInfo, write). delete_user2(UserInfo) -> @@ -376,8 +385,10 @@ trans(Fun, Args) -> {aborted, Reason} -> {error, Reason} end. - to_binary(B) when is_binary(B) -> B; to_binary(L) when is_list(L) -> iolist_to_binary(L). + +serialize_user_info(#user_info{user_id = {_, UserID}, superuser = Superuser}) -> + #{user_id => UserID, superuser => Superuser}. \ No newline at end of file diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index ff1b2161a..56ced0104 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -140,7 +140,8 @@ authenticate(#{password := Password} = Credential, ignore; Doc -> case check_password(Password, Doc, State) of - ok -> ok; + ok -> + {ok, #{superuser => superuser(Doc, State)}}; {error, {cannot_find_password_hash_field, PasswordHashField}} -> ?LOG(error, "['~s'] Can't find password hash field: ~s", [Unique, PasswordHashField]), {error, bad_username_or_password}; @@ -221,6 +222,11 @@ check_password(Password, end end. +superuser(Doc, #{superuser_field := SuperuserField}) -> + maps:get(SuperuserField, Doc, false); +superuser(_, _) -> + false. + hash(Algorithm, Password, Salt, prefix) -> emqx_passwd:hash(Algorithm, <>); hash(Algorithm, Password, Salt, suffix) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index f2a01e7e1..75a3392ec 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -112,15 +112,19 @@ authenticate(#{password := Password} = Credential, case emqx_resource:query(Unique, {sql, Query, Params, Timeout}) of {ok, _Columns, []} -> ignore; {ok, Columns, Rows} -> - %% TODO: Support superuser Selected = maps:from_list(lists:zip(Columns, Rows)), - check_password(Password, Selected, State); + case check_password(Password, Selected, State) of + ok -> + {ok, #{superuser => maps:get(<<"superuser">>, Selected, false)}}; + {error, Reason} -> + {error, Reason} + end; {error, _Reason} -> ignore end catch - error:Reason -> - ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Reason]), + error:Error -> + ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Error]), ignore end. @@ -135,17 +139,17 @@ destroy(#{'_unique' := Unique}) -> check_password(undefined, _Selected, _State) -> {error, bad_username_or_password}; check_password(Password, - #{password_hash := Hash}, + #{<<"password_hash">> := Hash}, #{password_hash_algorithm := bcrypt}) -> case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of true -> ok; false -> {error, bad_username_or_password} end; check_password(Password, - #{password_hash := Hash} = Selected, + #{<<"password_hash">> := Hash} = Selected, #{password_hash_algorithm := Algorithm, salt_position := SaltPosition}) -> - Salt = maps:get(salt, Selected, <<>>), + Salt = maps:get(<<"salt">>, Selected, <<>>), case Hash =:= emqx_authn_utils:hash(Algorithm, Password, Salt, SaltPosition) of true -> ok; false -> {error, bad_username_or_password} diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index b83e111c3..44c7f7185 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -18,6 +18,7 @@ -include("emqx_authn.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("epgsql/include/epgsql.hrl"). -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). @@ -98,15 +99,20 @@ authenticate(#{password := Password} = Credential, case emqx_resource:query(Unique, {sql, Query, Params}) of {ok, _Columns, []} -> ignore; {ok, Columns, Rows} -> - %% TODO: Support superuser - Selected = maps:from_list(lists:zip(Columns, Rows)), - check_password(Password, Selected, State); + NColumns = [Name || #column{name = Name} <- Columns], + Selected = maps:from_list(lists:zip(NColumns, Rows)), + case check_password(Password, Selected, State) of + ok -> + {ok, #{superuser => maps:get(<<"superuser">>, Selected, false)}}; + {error, Reason} -> + {error, Reason} + end; {error, _Reason} -> ignore end catch - error:Reason -> - ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Reason]), + error:Error -> + ?LOG(warning, "The following error occurred in '~s' during authentication: ~p", [Unique, Error]), ignore end. @@ -121,17 +127,17 @@ destroy(#{'_unique' := Unique}) -> check_password(undefined, _Selected, _State) -> {error, bad_username_or_password}; check_password(Password, - #{password_hash := Hash}, + #{<<"password_hash">> := Hash}, #{password_hash_algorithm := bcrypt}) -> case {ok, Hash} =:= bcrypt:hashpw(Password, Hash) of true -> ok; false -> {error, bad_username_or_password} end; check_password(Password, - #{password_hash := Hash} = Selected, + #{<<"password_hash">> := Hash} = Selected, #{password_hash_algorithm := Algorithm, salt_position := SaltPosition}) -> - Salt = maps:get(salt, Selected, <<>>), + Salt = maps:get(<<"salt">>, Selected, <<>>), case Hash =:= emqx_authn_utils:hash(Algorithm, Password, Salt, SaltPosition) of true -> ok; false -> {error, bad_username_or_password} diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 5d6e579ac..0c2696c0e 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -124,7 +124,13 @@ authenticate(#{password := Password} = Credential, NKey = binary_to_list(iolist_to_binary(replace_placeholders(Key, Credential))), case emqx_resource:query(Unique, {cmd, [Command, NKey | Fields]}) of {ok, Values} -> - check_password(Password, merge(Fields, Values), State); + Selected = merge(Fields, Values), + case check_password(Password, Selected, State) of + ok -> + {ok, #{superuser => maps:get("superuser", Selected, false)}}; + {error, Reason} -> + {error, Reason} + end; {error, Reason} -> ?LOG(error, "['~s'] Query failed: ~p", [Unique, Reason]), ignore @@ -166,8 +172,8 @@ check_fields(["password_hash" | More], false) -> check_fields(More, true); check_fields(["salt" | More], HasPassHash) -> check_fields(More, HasPassHash); -% check_fields(["is_superuser" | More], HasPassHash) -> -% check_fields(More, HasPassHash); +check_fields(["superuser" | More], HasPassHash) -> + check_fields(More, HasPassHash); check_fields([Field | _], _) -> error({unsupported_field, Field}). diff --git a/apps/emqx_authn/test/data/user-credentials.csv b/apps/emqx_authn/test/data/user-credentials.csv index 2543d39ca..0548308b7 100644 --- a/apps/emqx_authn/test/data/user-credentials.csv +++ b/apps/emqx_authn/test/data/user-credentials.csv @@ -1,3 +1,3 @@ -user_id,password_hash,salt -myuser3,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235 -myuser4,ee68c985a69208b6eda8c6c9b4c7c2d2b15ee2352cdd64a903171710a99182e8,ad773b5be9dd0613fe6c2f4d8c403139 +user_id,password_hash,salt,superuser +myuser3,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true +myuser4,ee68c985a69208b6eda8c6c9b4c7c2d2b15ee2352cdd64a903171710a99182e8,ad773b5be9dd0613fe6c2f4d8c403139,false diff --git a/apps/emqx_authn/test/data/user-credentials.json b/apps/emqx_authn/test/data/user-credentials.json index 169122bd2..e54501233 100644 --- a/apps/emqx_authn/test/data/user-credentials.json +++ b/apps/emqx_authn/test/data/user-credentials.json @@ -2,11 +2,13 @@ { "user_id":"myuser1", "password_hash":"c5e46903df45e5dc096dc74657610dbee8deaacae656df88a1788f1847390242", - "salt": "e378187547bf2d6f0545a3f441aa4d8a" + "salt": "e378187547bf2d6f0545a3f441aa4d8a", + "superuser": true }, { "user_id":"myuser2", "password_hash":"f4d17f300b11e522fd33f497c11b126ef1ea5149c74d2220f9a16dc876d4567b", - "salt": "6d3f9bd5b54d94b98adbcfe10b6d181f" + "salt": "6d3f9bd5b54d94b98adbcfe10b6d181f", + "superuser": false } ] diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index 9c4371838..bf6447aba 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -93,7 +93,7 @@ t_authenticator(_) -> ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, {before, ID1})), - + ?assertMatch({ok, [#{name := AuthenticatorName2}, #{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), ?assertEqual({error, {not_found, {authenticator, <<"nonexistent">>}}}, ?AUTH:move_authenticator(?CHAIN, ID2, {before, <<"nonexistent">>})), @@ -108,7 +108,7 @@ t_authenticate(_) -> listener => mqtt_tcp, username => <<"myuser">>, password => <<"mypass">>}, - ?assertEqual(ok, emqx_access_control:authenticate(ClientInfo)), + ?assertEqual({ok, #{superuser => false}}, emqx_access_control:authenticate(ClientInfo)), ?assertEqual(false, emqx_authn:is_enabled()), emqx_authn:enable(), ?assertEqual(true, emqx_authn:is_enabled()), diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index 7435deaa0..ddb2bb209 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -52,21 +52,27 @@ t_jwt_authenticator(_) -> JWS = generate_jws('hmac-based', Payload, <<"abcdef">>), ClientInfo = #{username => <<"myuser">>, password => JWS}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), + + Payload1 = #{<<"username">> => <<"myuser">>, <<"superuser">> => true}, + JWS1 = generate_jws('hmac-based', Payload1, <<"abcdef">>), + ClientInfo1 = #{username => <<"myuser">>, + password => JWS1}, + ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), ClientInfo2 = ClientInfo#{password => BadJWS}, - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ok)), + ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), %% secret_base64_encoded Config2 = Config#{secret => base64:encode(<<"abcdef">>), secret_base64_encoded => true}, ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)), - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]}, ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)), - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)), %% Expiration @@ -74,39 +80,39 @@ t_jwt_authenticator(_) -> , <<"exp">> => erlang:system_time(second) - 60}, JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>), ClientInfo3 = ClientInfo#{password => JWS3}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ok)), + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), Payload4 = #{ <<"username">> => <<"myuser">> , <<"exp">> => erlang:system_time(second) + 60}, JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>), ClientInfo4 = ClientInfo#{password => JWS4}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo4, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), %% Issued At Payload5 = #{ <<"username">> => <<"myuser">> , <<"iat">> => erlang:system_time(second) - 60}, JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>), ClientInfo5 = ClientInfo#{password => JWS5}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo5, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo5, ignored)), Payload6 = #{ <<"username">> => <<"myuser">> , <<"iat">> => erlang:system_time(second) + 60}, JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>), ClientInfo6 = ClientInfo#{password => JWS6}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ok)), + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ignored)), %% Not Before Payload7 = #{ <<"username">> => <<"myuser">> , <<"nbf">> => erlang:system_time(second) - 60}, JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>), ClientInfo7 = ClientInfo#{password => JWS7}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo7, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo7, ignored)), Payload8 = #{ <<"username">> => <<"myuser">> , <<"nbf">> => erlang:system_time(second) + 60}, JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>), ClientInfo8 = ClientInfo#{password => JWS8}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ok)), + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ignored)), ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), ok. @@ -128,8 +134,8 @@ t_jwt_authenticator2(_) -> JWS = generate_jws('public-key', Payload, PrivateKey), ClientInfo = #{username => <<"myuser">>, password => JWS}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), + ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ignored)), ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), ok. diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index fdcaf519d..d6425a89c 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -50,33 +50,36 @@ t_mnesia_authenticator(_) -> UserInfo = #{user_id => <<"myuser">>, password => <<"mypass">>}, - ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), - ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), + ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), + ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), ClientInfo = #{zone => external, username => <<"myuser">>, password => <<"mypass">>}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), ?AUTH:enable(), - ?assertEqual(ok, emqx_access_control:authenticate(ClientInfo)), + ?assertEqual({ok, #{superuser => false}}, emqx_access_control:authenticate(ClientInfo)), ClientInfo2 = ClientInfo#{username => <<"baduser">>}, - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ok)), + ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo2)), ClientInfo3 = ClientInfo#{password => <<"badpass">>}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ok)), + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo3)), UserInfo2 = UserInfo#{password => <<"mypass2">>}, - ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, UserInfo2)), + ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, UserInfo2)), ClientInfo4 = ClientInfo#{password => <<"mypass2">>}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo4, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), + + ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, #{superuser => true})), + ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo4, ignored)), ?assertEqual(ok, ?AUTH:delete_user(?CHAIN, ID, <<"myuser">>)), ?assertEqual({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), - ?assertEqual({ok, #{user_id => <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), + ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), @@ -104,10 +107,16 @@ t_import(_) -> ClientInfo1 = #{username => <<"myuser1">>, password => <<"mypassword1">>}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo1, ok)), - ClientInfo2 = ClientInfo1#{username => <<"myuser3">>, + ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), + + ClientInfo2 = ClientInfo1#{username => <<"myuser2">>, + password => <<"mypassword2">>}, + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), + + ClientInfo3 = ClientInfo1#{username => <<"myuser3">>, password => <<"mypassword3">>}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo2, ok)), + ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo3, ignored)), + ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), ok. @@ -131,11 +140,11 @@ t_multi_mnesia_authenticator(_) -> {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig2), - ?assertEqual({ok, #{user_id => <<"myuser">>}}, + ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID1, #{user_id => <<"myuser">>, password => <<"mypass1">>})), - ?assertEqual({ok, #{user_id => <<"myclient">>}}, + ?assertMatch({ok, #{user_id := <<"myclient">>}}, ?AUTH:add_user(?CHAIN, ID2, #{user_id => <<"myclient">>, password => <<"mypass2">>})), @@ -143,12 +152,12 @@ t_multi_mnesia_authenticator(_) -> ClientInfo1 = #{username => <<"myuser">>, clientid => <<"myclient">>, password => <<"mypass1">>}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo1, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo1, ignored)), ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo1, ok)), + ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo1, ignored)), ClientInfo2 = ClientInfo1#{password => <<"mypass2">>}, - ?assertEqual({stop, ok}, ?AUTH:authenticate(ClientInfo2, ok)), + ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), From e5892d16e5d0db62954e42756d4bc75203ee4de8 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Wed, 18 Aug 2021 18:24:52 +0800 Subject: [PATCH 15/20] feat(auth): support hot config --- apps/emqx_authn/include/emqx_authn.hrl | 1 - apps/emqx_authn/src/emqx_authn.erl | 104 ++++++++++------------ apps/emqx_authn/src/emqx_authn_api.erl | 50 +++++++---- apps/emqx_authn/test/emqx_authn_SUITE.erl | 6 +- 4 files changed, 82 insertions(+), 79 deletions(-) diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index f9ba7c3b5..c5a392fd0 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -26,7 +26,6 @@ { id :: binary() , name :: binary() , provider :: module() - , config :: map() , state :: map() }). diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 384830750..d6644cad5 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -76,60 +76,63 @@ %%------------------------------------------------------------------------------ pre_config_update({enable, Enable}, _OldConfig) -> - Enable; + {ok, Enable}; pre_config_update({create_authenticator, Config}, OldConfig) -> - OldConfig ++ [Config]; + {ok, OldConfig ++ [Config]}; pre_config_update({delete_authenticator, ID}, OldConfig) -> case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> error(Reason); + {error, Reason} -> {error, Reason}; {ok, #{name := Name}} -> - lists:filter(fun(#{<<"name">> := N}) -> - N =/= Name - end, OldConfig) + NewConfig = lists:filter(fun(#{<<"name">> := N}) -> + N =/= Name + end, OldConfig), + {ok, NewConfig} end; pre_config_update({update_authenticator, ID, Config}, OldConfig) -> case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> error(Reason); + {error, Reason} -> {error, Reason}; {ok, #{name := Name}} -> - lists:map(fun(#{<<"name">> := N} = C) -> - case N =:= Name of - true -> Config; - false -> C - end - end, OldConfig) + NewConfig = lists:map(fun(#{<<"name">> := N} = C) -> + case N =:= Name of + true -> Config; + false -> C + end + end, OldConfig), + {ok, NewConfig} end; pre_config_update({update_or_create_authenticator, ID, Config}, OldConfig) -> case lookup_authenticator(?CHAIN, ID) of {error, _Reason} -> OldConfig ++ [Config]; {ok, #{name := Name}} -> - lists:map(fun(#{<<"name">> := N} = C) -> - case N =:= Name of - true -> Config; - false -> C - end - end, OldConfig) + NewConfig = lists:map(fun(#{<<"name">> := N} = C) -> + case N =:= Name of + true -> Config; + false -> C + end + end, OldConfig), + {ok, NewConfig} end; -pre_config_update({move, ID, Position}, OldConfig) -> +pre_config_update({move_authenticator, ID, Position}, OldConfig) -> case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> error(Reason); + {error, Reason} -> {error, Reason}; {ok, #{name := Name}} -> {ok, Found, Part1, Part2} = split_by_name(Name, OldConfig), case Position of <<"top">> -> - [Found | Part1] ++ Part2; + {ok, [Found | Part1] ++ Part2}; <<"bottom">> -> - Part1 ++ Part2 ++ [Found]; + {ok, Part1 ++ Part2 ++ [Found]}; Before -> case binary:split(Before, <<":">>, [global]) of [<<"before">>, ID0] -> case lookup_authenticator(?CHAIN, ID0) of - {error, Reason} -> error(Reason); + {error, Reason} -> {error, Reason}; {ok, #{name := Name1}} -> - {ok, NFound, NPart1, NPart2} = split_by_name(Name1, Part1 + Part2), - NPart1 ++ [Found, NFound | NPart2] + {ok, NFound, NPart1, NPart2} = split_by_name(Name1, Part1 ++ Part2), + {ok, NPart1 ++ [Found, NFound | NPart2]} end; _ -> - error({invalid_parameter, position}) + {error, {invalid_parameter, position}} end end end. @@ -144,12 +147,9 @@ post_config_update({create_authenticator, #{<<"name">> := Name}}, NewConfig, _Ol N =:= Name end, NewConfig) of [Config] -> - case create_authenticator(?CHAIN, Config) of - {ok, _} -> ok; - {error, Reason} -> throw(Reason) - end; + create_authenticator(?CHAIN, Config); [_Config | _] -> - error(name_has_be_used) + {error, name_has_be_used} end; post_config_update({delete_authenticator, ID}, _NewConfig, _OldConfig) -> case delete_authenticator(?CHAIN, ID) of @@ -162,12 +162,9 @@ post_config_update({update_authenticator, ID, #{<<"name">> := Name}}, NewConfig, N =:= Name end, NewConfig) of [Config] -> - case update_authenticator(?CHAIN, ID, Config) of - {ok, _} -> ok; - {error, Reason} -> throw(Reason) - end; + update_authenticator(?CHAIN, ID, Config); [_Config | _] -> - error(name_has_be_used) + {error, name_has_be_used} end; post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig) -> case lists:filter( @@ -175,14 +172,11 @@ post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}}, N =:= Name end, NewConfig) of [Config] -> - case update_or_create_authenticator(?CHAIN, ID, Config) of - {ok, _} -> ok; - {error, Reason} -> throw(Reason) - end; + update_or_create_authenticator(?CHAIN, ID, Config); [_Config | _] -> - error(name_has_be_used) + {error, name_has_be_used} end; -post_config_update({move, ID, Position}, _NewConfig, _OldConfig) -> +post_config_update({move_authenticator, ID, Position}, _NewConfig, _OldConfig) -> NPosition = case Position of <<"top">> -> top; <<"bottom">> -> bottom; @@ -191,16 +185,13 @@ post_config_update({move, ID, Position}, _NewConfig, _OldConfig) -> [<<"before">>, ID0] -> {before, ID0}; _ -> - error({invalid_parameter, position}) + {error, {invalid_parameter, position}} end end, - case move_authenticator(?CHAIN, ID, NPosition) of - ok -> ok; - {error, Reason} -> throw(Reason) - end. + move_authenticator(?CHAIN, ID, NPosition). update_config(Path, ConfigRequest) -> - emqx_config:update(emqx_authn_schema, Path, ConfigRequest). + emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). enable() -> case emqx:hook('client.authenticate', {?MODULE, authenticate, []}) of @@ -522,7 +513,6 @@ do_create_authenticator(ChainID, AuthenticatorID, #{name := Name} = Config) -> Authenticator = #authenticator{id = AuthenticatorID, name = Name, provider = Provider, - config = Config, state = switch_version(State)}, {ok, Authenticator}; {error, Reason} -> @@ -570,8 +560,7 @@ update_or_create_authenticator(ChainID, AuthenticatorID, #{name := NewName} = Co case Provider:update(Config#{'_unique' => Unique}, State) of {ok, NewState} -> NewAuthenticator = Authenticator#authenticator{name = NewName, - config = Config, - state = switch_version(NewState)}, + state = switch_version(NewState)}, NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}), {ok, serialize_authenticator(NewAuthenticator)}; @@ -583,9 +572,8 @@ update_or_create_authenticator(ChainID, AuthenticatorID, #{name := NewName} = Co case NewProvider:create(Config#{'_unique' => Unique}) of {ok, NewState} -> NewAuthenticator = Authenticator#authenticator{name = NewName, - provider = NewProvider, - config = Config, - state = switch_version(NewState)}, + provider = NewProvider, + state = switch_version(NewState)}, NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}), _ = Provider:destroy(State), @@ -660,5 +648,7 @@ serialize_authenticators(Authenticators) -> [serialize_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators]. serialize_authenticator(#authenticator{id = ID, - config = Config}) -> - Config#{id => ID}. + name = Name, + provider = Provider, + state = State}) -> + #{id => ID, name => Name, provider => Provider, state => State}. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 97ed94a8b..20a8e2f7d 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -790,6 +790,7 @@ definitions() -> , minirest:ref(<<"password_based_mysql">>) , minirest:ref(<<"password_based_pgsql">>) , minirest:ref(<<"password_based_mongodb">>) + , minirest:ref(<<"password_based_redis">>) , minirest:ref(<<"password_based_http_server">>) ] } @@ -1292,7 +1293,7 @@ authentication(post, Request) -> {ok, Body, _} = cowboy_req:read_body(Request), case emqx_json:decode(Body, [return_maps]) of #{<<"enable">> := Enable} -> - emqx_authn:update_config([authentication, enable], {enable, Enable}), + {ok, _} = emqx_authn:update_config([authentication, enable], {enable, Enable}), {204}; _ -> serialize_error({missing_parameter, enable}) @@ -1305,20 +1306,28 @@ authenticators(post, Request) -> {ok, Body, _} = cowboy_req:read_body(Request), Config = emqx_json:decode(Body, [return_maps]), case emqx_authn:update_config([authentication, authenticators], {create_authenticator, Config}) of - ok -> - {204}; - {error, Reason} -> + {ok, #{post_config_update := #{emqx_authn := #{id := ID, name := Name}}, + raw_config := RawConfig}} -> + [RawConfig1] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], + {200, RawConfig1#{id => ID}}; + {error, {_, _, Reason}} -> serialize_error(Reason) end; authenticators(get, _Request) -> + RawConfig = get_raw_config([authentication, authenticators]), {ok, Authenticators} = emqx_authn:list_authenticators(?CHAIN), - {200, Authenticators}. + NAuthenticators = lists:zipwith(fun(#{<<"name">> := Name} = Config, #{id := ID, name := Name}) -> + Config#{id => ID} + end, RawConfig, Authenticators), + {200, NAuthenticators}. authenticators2(get, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), case emqx_authn:lookup_authenticator(?CHAIN, AuthenticatorID) of - {ok, Authenticator} -> - {200, Authenticator}; + {ok, #{id := ID, name := Name}} -> + RawConfig = get_raw_config([authentication, authenticators]), + [RawConfig1] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], + {200, RawConfig1#{id => ID}}; {error, Reason} -> serialize_error(Reason) end; @@ -1328,17 +1337,19 @@ authenticators2(put, Request) -> Config = emqx_json:decode(Body, [return_maps]), case emqx_authn:update_config([authentication, authenticators], {update_or_create_authenticator, AuthenticatorID, Config}) of - ok -> - {204}; - {error, Reason} -> + {ok, #{post_config_update := #{emqx_authn := #{id := ID, name := Name}}, + raw_config := RawConfig}} -> + [RawConfig0] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], + {200, RawConfig0#{id => ID}}; + {error, {_, _, Reason}} -> serialize_error(Reason) end; authenticators2(delete, Request) -> AuthenticatorID = cowboy_req:binding(id, Request), case emqx_authn:update_config([authentication, authenticators], {delete_authenticator, AuthenticatorID}) of - ok -> + {ok, _} -> {204}; - {error, Reason} -> + {error, {_, _, Reason}} -> serialize_error(Reason) end. @@ -1348,8 +1359,8 @@ move(post, Request) -> case emqx_json:decode(Body, [return_maps]) of #{<<"position">> := Position} -> case emqx_authn:update_config([authentication, authenticators], {move_authenticator, AuthenticatorID, Position}) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) + {ok, _} -> {204}; + {error, {_, _, Reason}} -> serialize_error(Reason) end; _ -> serialize_error({missing_parameter, position}) @@ -1361,10 +1372,8 @@ import_users(post, Request) -> case emqx_json:decode(Body, [return_maps]) of #{<<"filename">> := Filename} -> case emqx_authn:import_users(?CHAIN, AuthenticatorID, Filename) of - ok -> - {204}; - {error, Reason} -> - serialize_error(Reason) + ok -> {204}; + {error, Reason} -> serialize_error(Reason) end; _ -> serialize_error({missing_parameter, filename}) @@ -1435,6 +1444,11 @@ users2(delete, Request) -> serialize_error(Reason) end. +get_raw_config(ConfKeyPath) -> + %% TODO: call emqx_config:get_raw(ConfKeyPath) directly + NConfKeyPath = [atom_to_binary(Key, utf8) || Key <- ConfKeyPath], + emqx_map_lib:deep_get(NConfKeyPath, emqx_config:fill_defaults(emqx_config:get_raw([]))). + serialize_error({not_found, {authenticator, ID}}) -> {404, #{code => <<"NOT_FOUND">>, message => list_to_binary(io_lib:format("Authenticator '~s' does not exist", [ID]))}}; diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index bf6447aba..0be04d6cf 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -71,7 +71,7 @@ t_authenticator(_) -> secret => <<"abcdef">>, secret_base64_encoded => false, verify_claims => []}, - {ok, #{name := AuthenticatorName1, id := ID1, mechanism := jwt}} = ?AUTH:update_authenticator(?CHAIN, ID1, AuthenticatorConfig2), + {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:update_authenticator(?CHAIN, ID1, AuthenticatorConfig2), ID2 = <<"random">>, ?assertEqual({error, {not_found, {authenticator, ID2}}}, ?AUTH:update_authenticator(?CHAIN, ID2, AuthenticatorConfig2)), @@ -79,9 +79,9 @@ t_authenticator(_) -> AuthenticatorName2 = <<"myauthenticator2">>, AuthenticatorConfig3 = AuthenticatorConfig2#{name => AuthenticatorName2}, - {ok, #{name := AuthenticatorName2, id := ID2, secret := <<"abcdef">>}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3), + {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3), ?assertMatch({ok, #{name := AuthenticatorName2}}, ?AUTH:lookup_authenticator(?CHAIN, ID2)), - {ok, #{name := AuthenticatorName2, id := ID2, secret := <<"fedcba">>}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3#{secret := <<"fedcba">>}), + {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3#{secret := <<"fedcba">>}), ?assertMatch({ok, #{id := ?CHAIN, authenticators := [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}}, ?AUTH:lookup_chain(?CHAIN)), ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), From 178d1006e1788ad8c04932d3b9e0e8b44d3f331a Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 19 Aug 2021 09:23:38 +0800 Subject: [PATCH 16/20] chore(match): reduce the risk of crash --- apps/emqx/src/emqx_channel.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 93d5e2c37..5988e03e5 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1302,13 +1302,13 @@ authenticate(?AUTH_PACKET(_, #{'Authentication-Method' := AuthMethod} = Properti do_authenticate(#{auth_method := AuthMethod} = Credential, #channel{clientinfo = ClientInfo} = Channel) -> Properties = #{'Authentication-Method' => AuthMethod}, case emqx_access_control:authenticate(Credential) of - {ok, #{superuser := Superuser}} -> + {ok, Result} -> {ok, Properties, - Channel#channel{clientinfo = ClientInfo#{is_superuser => Superuser}, + Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(superuser, Result, false)}, auth_cache = #{}}}; - {ok, #{superuser := Superuser}, AuthData} -> + {ok, Result, AuthData} -> {ok, Properties#{'Authentication-Data' => AuthData}, - Channel#channel{clientinfo = ClientInfo#{is_superuser => Superuser}, + Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(superuser, Result, false)}, auth_cache = #{}}}; {continue, AuthCache} -> {continue, Properties, Channel#channel{auth_cache = AuthCache}}; From d409cb83ff9e79e6cfc126523896da00d355aa60 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 19 Aug 2021 10:26:50 +0800 Subject: [PATCH 17/20] chore(authn): update tag of esasl --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 1abaef868..8cfc3cf7c 100644 --- a/rebar.config +++ b/rebar.config @@ -63,7 +63,7 @@ , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}} - , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.1.0"}}} + , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} ]}. {xref_ignores, From 87051c0e533f7da5125e83080c2bd892fc7d0088 Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 19 Aug 2021 14:40:36 +0800 Subject: [PATCH 18/20] test(authn): fix test case --- apps/emqx/test/emqx_access_control_SUITE.erl | 2 +- apps/emqx/test/emqx_channel_SUITE.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_ctx.erl | 2 +- apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index d459d28b2..8cfa17523 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -33,7 +33,7 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]). t_authenticate(_) -> - ?assertMatch(ok, emqx_access_control:authenticate(clientinfo())). + ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). t_authorize(_) -> Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>), diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index be7c94ede..dfbe56916 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -181,7 +181,7 @@ init_per_suite(Config) -> %% Access Control Meck ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect(emqx_access_control, authenticate, - fun(_) -> ok end), + fun(_) -> {ok, #{superuser => false}} end), ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end), %% Broker Meck ok = meck:new(emqx_broker, [passthrough, no_history, no_link]), diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl index 22391d0b6..b5de6cb9a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -73,7 +73,7 @@ authenticate(_Ctx = #{auth := ChainId}, ClientInfo0) -> chain_id => ChainId }, case emqx_access_control:authenticate(ClientInfo) of - ok -> + {ok, _} -> {ok, mountpoint(ClientInfo)}; {error, Reason} -> {error, Reason} diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl index f14d6cb73..f96fe714c 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl @@ -87,7 +87,7 @@ init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> ClientInfo = clientinfo(Lwm2mState), _ = run_hooks('client.connect', [conninfo(Lwm2mState)], undefined), case emqx_access_control:authenticate(ClientInfo) of - ok -> + {ok, _} -> _ = run_hooks('client.connack', [conninfo(Lwm2mState), success], undefined), %% FIXME: From e6c01cb6e69f9704fb3760045c15222a2ecf07ef Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 19 Aug 2021 15:01:44 +0800 Subject: [PATCH 19/20] fix(undefined function): fix undefined function --- .../src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 98cdb8c26..2d433d408 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -177,7 +177,7 @@ update_user(UserID, User, undefined -> UserInfo1; Password -> - {StoredKey, ServerKey, Salt} = esasl_scram:generate_user_credential(Password, State), + {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State), UserInfo1#user_info{stored_key = StoredKey, server_key = ServerKey, salt = Salt} @@ -239,7 +239,7 @@ check_client_final_message(Bin, #{superuser := Superuser} = Cache, #{algorithm : end. add_user(UserID, Password, Superuser, State) -> - {StoredKey, ServerKey, Salt} = esasl_scram:generate_user_credential(Password, State), + {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State), UserInfo = #user_info{user_id = UserID, stored_key = StoredKey, server_key = ServerKey, From f0ba6af660e6683a5ba2e779b3fc4e866935807e Mon Sep 17 00:00:00 2001 From: zhouzb Date: Thu, 19 Aug 2021 15:58:09 +0800 Subject: [PATCH 20/20] chore(authn): fix dialyzer --- apps/emqx/src/emqx_config.erl | 4 ++-- apps/emqx_authn/src/emqx_authn.erl | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 056929123..6441afe69 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -101,9 +101,9 @@ }. %% raw_config() is the config that is NOT parsed and tranlated by hocon schema --type raw_config() :: #{binary() => term()} | undefined. +-type raw_config() :: #{binary() => term()} | list() | undefined. %% config() is the config that is parsed and tranlated by hocon schema --type config() :: #{atom() => term()} | undefined. +-type config() :: #{atom() => term()} | list() | undefined. -type app_envs() :: [proplists:property()]. %% @doc For the given path, get root value enclosed in a single-key map. diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index d6644cad5..571d76cc7 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -314,9 +314,9 @@ list_users(ChainID, AuthenticatorID) -> %%-------------------------------------------------------------------- init(_Opts) -> - ets:new(?CHAIN_TAB, [ named_table, set, public - , {keypos, #chain.id} - , {read_concurrency, true}]), + _ = ets:new(?CHAIN_TAB, [ named_table, set, public + , {keypos, #chain.id} + , {read_concurrency, true}]), {ok, #{}}. handle_call({create_chain, ID}, _From, State) ->