diff --git a/.github/workflows/apps_version_check.yaml b/.github/workflows/apps_version_check.yaml index 2ef18934f..59d6aebff 100644 --- a/.github/workflows/apps_version_check.yaml +++ b/.github/workflows/apps_version_check.yaml @@ -19,6 +19,8 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 # need full history + - name: fix-git-unsafe-repository + run: git config --global --add safe.directory /__w/emqx/emqx - name: Check relup (ce) if: endsWith(github.repository, 'emqx') run: ./scripts/update-appup.sh emqx --check diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index a10e72258..529faa559 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -29,6 +29,8 @@ jobs: else echo "EMQX_NAME=emqx" >> $GITHUB_ENV fi + - name: fix-git-unsafe-repository + run: git config --global --add safe.directory /__w/emqx/emqx - name: build zip packages run: make ${EMQX_NAME}-zip - name: build deb/rpm packages diff --git a/.github/workflows/run_cts_tests.yaml b/.github/workflows/run_cts_tests.yaml index 84aa301c4..6b05a014e 100644 --- a/.github/workflows/run_cts_tests.yaml +++ b/.github/workflows/run_cts_tests.yaml @@ -45,6 +45,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | docker exec -i erlang sh -c "make ensure-rebar3" @@ -115,6 +117,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | printenv | grep "^EMQX_" > .env @@ -197,6 +201,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | printenv | grep "^EMQX_" > .env @@ -268,6 +274,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | export EMQX_AUTH__PGSQL__USERNAME=root \ @@ -391,6 +399,8 @@ jobs: if make emqx-ee --dry-run > /dev/null 2>&1; then docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store" fi + - name: fix-git-unsafe-repository + run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx" - name: run test cases run: | export EMQX_AUTH__REIDS__PASSWORD=public diff --git a/CHANGES-4.3.md b/CHANGES-4.3.md index 5f904913f..e948b7163 100644 --- a/CHANGES-4.3.md +++ b/CHANGES-4.3.md @@ -14,28 +14,32 @@ File format: ### Enhancements +* Refactored `bin/emqx` help messages. +* Upgrade script refuses upgrade from incompatible versions. (e.g. hot upgrade from 4.3 to 4.4 will fail fast). * Made possible for EMQX to boot from a Linux directory which has white spaces in its path. * Add support for JWT authorization [#7596] Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists. * Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes) * Return a client_identifier_not_valid error when username is empty and username_as_clientid is set to true [#7862] -* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/4 [#7894] +* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/3, date_to_unix_ts/4 [#7894] * Add proto_name and proto_ver fields for $event/client_disconnected event. * Mnesia auth/acl http api support multiple condition queries. * Inflight QoS1 Messages for shared topics are now redispatched to another alive subscribers upon chosen subscriber session termination. +* Make auth metrics name more understandable. +* Allow emqx_management http listener binding to specific interface [#8005] +* Add rule-engine function float2str/2, user can specify the float output precision [#7991] ### Bug fixes * List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions. * SSL closed error bug fixed for redis client. * Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message * Rule-engine function hexstr2bin/1 support half byte [#7977] -* Add rule-engine function float2str/2, user can specify the float output precision [#7991] -* Shared message delivery when all alive shared subs have full inflight - +* Shared message delivery when all alive shared subs have full inflight [#7984] * Improved resilience against autocluster partitioning during cluster startup. [#7876] [ekka-158](https://github.com/emqx/ekka/pull/158) * Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979] +* Fix `node_dump` variable sourcing. [#8026] ## v4.3.14 diff --git a/apps/emqx_auth_http/include/emqx_auth_http.hrl b/apps/emqx_auth_http/include/emqx_auth_http.hrl index b62f02bcd..0eaa59daf 100644 --- a/apps/emqx_auth_http/include/emqx_auth_http.hrl +++ b/apps/emqx_auth_http/include/emqx_auth_http.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_http). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_http/src/emqx_auth_http.app.src b/apps/emqx_auth_http/src/emqx_auth_http.app.src index fd8d1e046..fba56740f 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.app.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_http, [{description, "EMQ X Authentication/ACL with HTTP API"}, - {vsn, "4.3.5"}, % strict semver, bump manually! + {vsn, "4.3.6"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_http_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.appup.src b/apps/emqx_auth_http/src/emqx_auth_http.appup.src index 256bd7566..519604d24 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.appup.src +++ b/apps/emqx_auth_http/src/emqx_auth_http.appup.src @@ -1,27 +1,38 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.4", - [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]} - ]}, + [{"4.3.5", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, + {"4.3.4", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{apply,{application,stop,[emqx_auth_http]}}, {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-1]">>, [{restart_application,emqx_auth_http}]}, {<<".*">>,[]}], - [{"4.3.4", - [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}]}, + [{"4.3.5", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, + {"4.3.4", + [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]}, {"4.3.3", [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]}, {"4.3.2", [{apply,{application,stop,[emqx_auth_http]}}, {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]}, {<<"4.3.[0-1]">>, diff --git a/apps/emqx_auth_http/src/emqx_auth_http.erl b/apps/emqx_auth_http/src/emqx_auth_http.erl index f97276849..98a897a8c 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http.erl @@ -30,22 +30,16 @@ ]). %% Callbacks --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path}, super := SuperParams}) -> case authenticate(AuthParms, ClientInfo) of {ok, 200, <<"ignore">>} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {ok, 200, Body} -> - emqx_metrics:inc(?AUTH_METRICS(success)), IsSuperuser = is_superuser(SuperParams, ClientInfo), {stop, AuthResult#{is_superuser => IsSuperuser, auth_result => success, @@ -54,12 +48,10 @@ check(ClientInfo, AuthResult, #{auth := AuthParms = #{path := Path}, {ok, Code, _Body} -> ?LOG(error, "Deny connection from path: ~s, response http code: ~p", [Path, Code]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => http_to_connack_error(Code), anonymous => false}}; {error, Error} -> ?LOG(error, "Request auth path: ~s, error: ~p", [Path, Error]), - emqx_metrics:inc(?AUTH_METRICS(failure)), %%FIXME later: server_unavailable is not right. {stop, AuthResult#{auth_result => server_unavailable, anonymous => false}} diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index fb23f9c0a..9e4b842d8 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -112,7 +112,6 @@ load_hooks() -> case application:get_env(?APP, auth_req) of undefined -> ok; {ok, AuthReq} -> - ok = emqx_auth_http:register_metrics(), PoolOpts = proplists:get_value(pool_opts, AuthReq), PoolName = proplists:get_value(pool_name, AuthReq), {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts), @@ -160,4 +159,3 @@ path(#{path := ""}) -> "/"; path(#{path := Path}) -> Path. - diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src index a159d2ce4..b94159225 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src @@ -1,13 +1,9 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.2", - [{restart_application,emqx_auth_jwt}]}, - {<<"4\\.3\\.[0-1]">>, + [{<<"4\\.3\\.[0-2]">>, [{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}], - [{"4.3.2", - [{restart_application,emqx_auth_jwt}]}, - {<<"4\\.3\\.[0-1]">>, + [{<<"4\\.3\\.[0-2]">>, [{restart_application,emqx_auth_jwt}]}, {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl index b900f2bed..040f9b629 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt.erl @@ -21,28 +21,11 @@ -logger_header("[JWT]"). --export([ register_metrics/0 - , check_auth/3 +-export([ check_auth/3 , check_acl/5 , description/0 ]). --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). - --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - %%-------------------------------------------------------------------- %% Authentication callbacks %%-------------------------------------------------------------------- @@ -50,17 +33,16 @@ register_metrics() -> check_auth(ClientInfo, AuthResult, #{from := From, checklists := Checklists}) -> case maps:find(From, ClientInfo) of error -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {ok, undefined} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {ok, Token} -> case emqx_auth_jwt_svr:verify(Token) of {error, not_found} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, not_token} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, Reason} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => Reason, anonymous => false}}; {ok, Claims} -> {stop, maps:merge(AuthResult, verify_claims(Checklists, Claims, ClientInfo))} @@ -121,10 +103,8 @@ verify_acl(ClientInfo, [AclTopic | AclTopics], Topic) -> verify_claims(Checklists, Claims, ClientInfo) -> case do_verify_claims(feedvar(Checklists, ClientInfo), Claims) of {error, Reason} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), #{auth_result => Reason, anonymous => false}; ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), #{auth_result => success, anonymous => false, jwt_claims => Claims} end. diff --git a/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl b/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl index c94e8aa3c..d4e0e4960 100644 --- a/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl +++ b/apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl @@ -32,7 +32,6 @@ start(_Type, _Args) -> {ok, Sup} = supervisor:start_link({local, ?MODULE}, ?MODULE, []), {ok, _} = start_auth_server(jwks_svr_options()), - ok = emqx_auth_jwt:register_metrics(), AuthEnv = auth_env(), _ = emqx:hook('client.authenticate', {emqx_auth_jwt, check_auth, [AuthEnv]}), diff --git a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl index 97a7d0cfc..0c573e0e2 100644 --- a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl +++ b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_ldap). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index a072234d7..6875fca30 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_ldap, [{description, "EMQ X Authentication/ACL with LDAP"}, - {vsn, "4.3.4"}, % strict semver, bump manually! + {vsn, "4.3.5"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_ldap_sup]}, {applications, [kernel,stdlib,eldap2,ecpool]}, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src index ea44d7a1a..fbb59a176 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src @@ -1,40 +1,29 @@ -%% -*-: erlang -*- +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ {"4.3.3", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, - {"4.3.0", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {"4.3.1", - [ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, + [{<<"4\\.3\\.[3-4]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ], - [ {"4.3.3", []}, - {"4.3.0", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {"4.3.1", - [ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []} - , {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{<<"4\\.3\\.[3-4]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []} - , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} - ] -}. + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[0-1]">>, + [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl index f2b6140a2..da932c2fe 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.erl @@ -26,17 +26,12 @@ -import(emqx_auth_ldap_cli, [search/3]). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 , prepare_filter/4 , replace_vars/2 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{username := Username, password := Password}, AuthResult, State = #{password_attr := PasswdAttr, bind_as_user := BindAsUserRequired, pool := Pool}) -> CheckResult = @@ -63,12 +58,10 @@ check(ClientInfo = #{username := Username, password := Password}, AuthResult, end, case CheckResult of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{auth_result => success, anonymous => false}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, ResultCode} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), ?LOG(error, "[LDAP] Auth from ldap failed: ~p", [ResultCode]), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl index 63fde463b..14c31f1bd 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl @@ -49,7 +49,6 @@ stop(_State) -> ok. load_auth_hook(DeviceDn) -> - ok = emqx_auth_ldap:register_metrics(), Params = maps:from_list(DeviceDn), emqx:hook('client.authenticate', fun emqx_auth_ldap:check/3, [Params#{pool => ?APP}]). diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl index 5de24d913..8bf7d532b 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl @@ -48,7 +48,9 @@ init_per_group(GrpName, Cfg) -> Cfg. end_per_group(_GrpName, _Cfg) -> - emqx_ct_helpers:stop_apps([emqx_auth_ldap]). + emqx_ct_helpers:stop_apps([emqx_auth_ldap]), + %% clear the application envs to avoid cross-suite testcase failure + application:unload(emqx_auth_ldap). %%-------------------------------------------------------------------- %% Cases diff --git a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl index 4f56ab32a..99f733d78 100644 --- a/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl @@ -40,7 +40,9 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_auth_ldap]). + emqx_ct_helpers:stop_apps([emqx_auth_ldap]), + %% clear the application envs to avoid cross-suite testcase failure + application:unload(emqx_auth_ldap). check_auth(_) -> MqttUser1 = #{clientid => <<"mqttuser1">>, diff --git a/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl b/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl index 2d5a6eb6d..f8569ef8a 100644 --- a/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl +++ b/apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl @@ -41,15 +41,3 @@ }). -type(acl_record() :: {acl_target(), emqx_topic:topic(), action(), access(), created_at()}). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src index a9e47eb65..a849002d6 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src @@ -2,9 +2,11 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{<<"4\\.3\\.[5-6]">>, - [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]}, + [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, + {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]">>, [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, @@ -27,7 +29,9 @@ {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{<<"4\\.3\\.[5-6]">>, - [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, + [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-3]">>, diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl index 386adba4b..e14de3cf5 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl @@ -27,7 +27,6 @@ -define(TABLE, emqx_user). %% Auth callbacks -export([ init/1 - , register_metrics/0 , check/3 , description/0 ]). @@ -51,10 +50,6 @@ init(#{clientid_list := ClientidList, username_list := UsernameList}) -> ok = ekka_mnesia:copy_table(?TABLE, disc_copies). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - hash_type() -> application:get_env(emqx_auth_mnesia, password_hash, sha256). @@ -67,17 +62,14 @@ check(ClientInfo = #{ clientid := Clientid end), case ets:select(?TABLE, MatchSpec) of [] -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; List -> case match_password(NPassword, HashType, List) of false -> Info = maps:without([password], ClientInfo), ?LOG(info, "[Mnesia] Auth from mnesia failed: ~p", [Info]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{anonymous => false, auth_result => password_error}}; _ -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{anonymous => false, auth_result => success}} end end. diff --git a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl index 467b6cf7c..ee38b7165 100644 --- a/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl +++ b/apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl @@ -56,7 +56,6 @@ load_auth_hook() -> ClientidList = application:get_env(?APP, clientid_list, []), UsernameList = application:get_env(?APP, username_list, []), ok = emqx_auth_mnesia:init(#{clientid_list => ClientidList, username_list => UsernameList}), - ok = emqx_auth_mnesia:register_metrics(), Params = #{hash_type => emqx_auth_mnesia:hash_type()}, emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]). diff --git a/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl b/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl index 69044e920..68a97ad33 100644 --- a/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl +++ b/apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl @@ -1,4 +1,3 @@ - -define(APP, emqx_auth_mongo). -define(DEFAULT_SELECTORS, [{<<"username">>, <<"%u">>}]). @@ -14,15 +13,3 @@ -record(aclquery, {collection = <<"mqtt_acl">>, selector = {<<"username">>, <<"%u">>}}). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src index 3090656f7..2b6fedbb5 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mongo, [{description, "EMQ X Authentication/ACL with MongoDB"}, - {vsn, "4.3.3"}, % strict semver, bump manually! + {vsn, "4.3.4"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mongo_sup]}, {applications, [kernel,stdlib,mongodb,ecpool]}, diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src index a12cb37b1..1907b7fa7 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src @@ -1,10 +1,7 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.2", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.3.1", + [{<<"4\\.3\\.[1-3]">>, [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, {"4.3.0", @@ -12,10 +9,7 @@ {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.2", - [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, - {"4.3.1", + [{<<"4\\.3\\.[1-3]">>, [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]}, {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]}, {"4.3.0", diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl index 0df342295..bfb911707 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo.erl @@ -23,8 +23,7 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). @@ -39,20 +38,15 @@ , available/3 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, Env = #{authquery := AuthQuery, superquery := SuperQuery}) -> #authquery{collection = Collection, field = Fields, hash = HashType, selector = Selector} = AuthQuery, Pool = maps:get(pool, Env, ?APP), case query(Pool, Collection, maps:from_list(replvars(Selector, ClientInfo))) of - undefined -> emqx_metrics:inc(?AUTH_METRICS(ignore)); + undefined -> ok; {error, Reason} -> ?LOG(error, "[MongoDB] Can't connect to MongoDB server: ~0p", [Reason]), - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => not_authorized, anonymous => false}}; UserMap -> Result = case [maps:get(Field, UserMap, undefined) || Field <- Fields] of @@ -64,13 +58,11 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case Result of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, Error} -> ?LOG(error, "[MongoDB] check auth fail: ~p", [Error]), - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => Error, anonymous => false}} end end. diff --git a/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl b/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl index 92c025621..a63aa8193 100644 --- a/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl +++ b/apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl @@ -68,7 +68,6 @@ safe_start() -> reg_authmod(AuthQuery) -> case emqx_auth_mongo:available(?APP, AuthQuery) of ok -> - emqx_auth_mongo:register_metrics(), HookFun = fun emqx_auth_mongo:check/3, HookOptions = #{authquery => AuthQuery, superquery => undefined, pool => ?APP}, case r(super_query, application:get_env(?APP, super_query, undefined)) of @@ -122,4 +121,3 @@ r(auth_query, Config) -> r(acl_query, Config) -> #aclquery{collection = list_to_binary(get_value(collection, Config, "mqtt_acl")), selector = get_value(selector, Config, [?DEFAULT_SELECTORS])}. - diff --git a/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl b/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl index 56da35401..b7c185fcf 100644 --- a/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl +++ b/apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_mysql). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 6df4545fd..a0ddc4dd4 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_mysql, [{description, "EMQ X Authentication/ACL with MySQL"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_mysql_sup]}, {applications, [kernel,stdlib,mysql,ecpool]}, diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src index 88ff1b38e..a116d7dbb 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src @@ -1,16 +1,19 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.1", []}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_mysql,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] }. diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl index 17d618838..31d9a007f 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.erl @@ -22,17 +22,12 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/types.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). -define(EMPTY(Username), (Username =:= undefined orelse Username =:= <<>>)). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, #{auth_query := {AuthSql, AuthParams}, super_query := SuperQuery, @@ -51,15 +46,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {error, ResultCode} -> ?LOG(error, "[MySQL] Auth from mysql failed: ~p", [ResultCode]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -88,4 +81,3 @@ check_pass(Password, HashType) -> end. description() -> "Authentication with MySQL". - diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl b/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl index 716cb7a7a..e58f62a90 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl @@ -50,7 +50,6 @@ stop(_State) -> ok. load_auth_hook(AuthQuery) -> - ok = emqx_auth_mysql:register_metrics(), SuperQuery = parse_query(application:get_env(?APP, super_query, undefined)), {ok, HashType} = application:get_env(?APP, password_hash), Params = #{auth_query => AuthQuery, diff --git a/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl b/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl index 92b971667..565aa4668 100644 --- a/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl +++ b/apps/emqx_auth_pgsql/include/emqx_auth_pgsql.hrl @@ -1,13 +1 @@ -define(APP, emqx_auth_pgsql). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index e1a1c752c..c351ed13e 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,5 +1,5 @@ {deps, - [{epgsql, {git, "https://github.com/epgsql/epgsql.git", {tag, "4.4.0"}}} + [ ]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src index 75175a814..35cae1622 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_pgsql, [{description, "EMQ X Authentication/ACL with PostgreSQL"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_pgsql_sup]}, {applications, [kernel,stdlib,epgsql,ecpool]}, diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src index 1c9a0c572..494cc94c9 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.appup.src @@ -1,16 +1,11 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"4.3.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, - {"4.3.0", - [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, + [{<<"4\\.3\\.[0-2]">>, + %% restart it due to epgsql upgraded from 4.4.0 to 4.6.0 + %% in emqx_auth_pgsql:v4.3.3 + [{restart_application,emqx_auth_pgsql}]}, {<<".*">>,[]}], - [{"4.3.1", []}, - {"4.3.0", - [{load_module,emqx_auth_pgsql_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_acl_pgsql,brutal_purge,soft_purge,[]}]}, - {<<".*">>,[]}] -}. + [{<<"4\\.3\\.[0-2]">>, + [{restart_application,emqx_auth_pgsql}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl index f38552085..f673e07e4 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql.erl @@ -21,15 +21,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - %%-------------------------------------------------------------------- %% Auth Module Callbacks %%-------------------------------------------------------------------- @@ -50,15 +45,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - emqx_metrics:inc(?AUTH_METRICS(success)), {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo), anonymous => false, auth_result => success}}; {error, not_found} -> - emqx_metrics:inc(?AUTH_METRICS(ignore)), ok; + ok; {error, ResultCode} -> ?LOG(error, "[Postgres] Auth from pgsql failed: ~p", [ResultCode]), - emqx_metrics:inc(?AUTH_METRICS(failure)), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -88,4 +81,3 @@ check_pass(Password, HashType) -> end. description() -> "Authentication with PostgreSQL". - diff --git a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl index c658c4d11..dbfeb8423 100644 --- a/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl +++ b/apps/emqx_auth_pgsql/src/emqx_auth_pgsql_app.erl @@ -42,7 +42,6 @@ start(_StartType, _StartArgs) -> super_query => SuperQuery, hash_type => HashType, pool => ?APP}, - ok = emqx_auth_pgsql:register_metrics(), ok = emqx:hook('client.authenticate', fun emqx_auth_pgsql:check/3, [AuthEnv]) end), if_enabled(acl_query, fun(AclQuery) -> @@ -59,4 +58,3 @@ if_enabled(Par, Fun) -> {ok, Query} -> Fun(parse_query(Par, Query)); undefined -> ok end. - diff --git a/apps/emqx_auth_redis/include/emqx_auth_redis.hrl b/apps/emqx_auth_redis/include/emqx_auth_redis.hrl index fe488c37f..075d649ec 100644 --- a/apps/emqx_auth_redis/include/emqx_auth_redis.hrl +++ b/apps/emqx_auth_redis/include/emqx_auth_redis.hrl @@ -1,14 +1 @@ - -define(APP, emqx_auth_redis). - --record(auth_metrics, { - success = 'client.auth.success', - failure = 'client.auth.failure', - ignore = 'client.auth.ignore' - }). - --define(METRICS(Type), tl(tuple_to_list(#Type{}))). --define(METRICS(Type, K), #Type{}#Type.K). - --define(AUTH_METRICS, ?METRICS(auth_metrics)). --define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)). diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index ea6aaefcf..e9e37a463 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_auth_redis, [{description, "EMQ X Authentication/ACL with Redis"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_auth_redis_sup]}, {applications, [kernel,stdlib,eredis,eredis_cluster,ecpool]}, diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src b/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src index 83f5d46be..9036d77a8 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.appup.src @@ -1,16 +1,19 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.1", [ - %% There are only changes to the schema file, so we don't need - %% any commands here. - ]}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.1", []}, + [{<<"4\\.3\\.[1-2]">>, + [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}]}, {"4.3.0", [{load_module,emqx_auth_redis_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_auth_redis,brutal_purge,soft_purge,[]}, {load_module,emqx_acl_redis,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}] }. diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.erl b/apps/emqx_auth_redis/src/emqx_auth_redis.erl index 04d3542f0..d432e012b 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.erl @@ -21,15 +21,10 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). --export([ register_metrics/0 - , check/3 +-export([ check/3 , description/0 ]). --spec(register_metrics() -> ok). -register_metrics() -> - lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS). - check(ClientInfo = #{password := Password}, AuthResult, #{auth_cmd := AuthCmd, super_cmd := SuperCmd, @@ -52,15 +47,13 @@ check(ClientInfo = #{password := Password}, AuthResult, end, case CheckPass of ok -> - ok = emqx_metrics:inc(?AUTH_METRICS(success)), IsSuperuser = is_superuser(Pool, Type, SuperCmd, ClientInfo, Timeout), {stop, AuthResult#{is_superuser => IsSuperuser, anonymous => false, auth_result => success}}; {error, not_found} -> - ok = emqx_metrics:inc(?AUTH_METRICS(ignore)); + ok; {error, ResultCode} -> - ok = emqx_metrics:inc(?AUTH_METRICS(failure)), ?LOG(error, "[Redis] Auth from redis failed: ~p", [ResultCode]), {stop, AuthResult#{auth_result => ResultCode, anonymous => false}} end. @@ -82,4 +75,3 @@ check_pass(Password, HashType) -> ok -> ok; {error, _Reason} -> {error, not_authorized} end. - diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl index c6de8b80a..cbea448f9 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_app.erl @@ -49,7 +49,6 @@ load_auth_hook(AuthCmd) -> timeout => Timeout, type => Type, pool => ?APP}, - ok = emqx_auth_redis:register_metrics(), emqx:hook('client.authenticate', fun emqx_auth_redis:check/3, [Config]). load_acl_hook(AclCmd) -> @@ -66,4 +65,3 @@ if_cmd_enabled(Par, Fun) -> {ok, Cmd} -> Fun(Cmd); undefined -> ok end. - diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index fe566ae52..41c19f752 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.7"}, %% 4.3.3 is used by ee + {vsn, "4.3.8"}, %% 4.3.3 is used by ee {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, diff --git a/apps/emqx_exproto/src/emqx_exproto.appup.src b/apps/emqx_exproto/src/emqx_exproto.appup.src index 0da87e289..33bd4386c 100644 --- a/apps/emqx_exproto/src/emqx_exproto.appup.src +++ b/apps/emqx_exproto/src/emqx_exproto.appup.src @@ -1,14 +1,9 @@ %% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [ - {"4.3.6", - [ %% There are only changes to the schema file, so we don't need any - %% commands here - ]}, - {<<"4\\.3\\.[4-5]">>, - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, - {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, + [{<<"4\\.3\\.[6-7]">>, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-5]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, @@ -17,11 +12,9 @@ {load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.6", []}, - {<<"4\\.3\\.[4-5]">>, - [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, - {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, - {<<"4\\.3\\.[2-3]">>, + [{<<"4\\.3\\.[6-7]">>, + [{load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, + {<<"4\\.3\\.[2-5]">>, [{load_module,emqx_exproto_conn,brutal_purge,soft_purge,[]}, {load_module,emqx_exproto_channel,brutal_purge,soft_purge,[]}]}, {<<"4\\.3\\.[0-1]">>, diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 0e39b40a7..dce3b36e1 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -299,8 +299,6 @@ handle_call({auth, RequestedClientInfo, Password}, case emqx_access_control:authenticate(ClientInfo1#{password => Password}) of {ok, AuthResult} -> emqx_logger:set_metadata_clientid(ClientId), - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), NClientInfo = maps:merge(ClientInfo1, AuthResult), NChannel = Channel1#channel{clientinfo = NClientInfo}, case emqx_cm:open_session(true, NClientInfo, NConnInfo) of @@ -424,9 +422,6 @@ terminate(Reason, Channel) -> Req = #{reason => stringfy(Reason)}, try_dispatch(on_socket_closed, wrap(Req), Channel). -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. - packet_to_message(Topic, Qos, Payload, #channel{ conninfo = #{proto_ver := ProtoVer}, diff --git a/apps/emqx_management/priv/emqx_management.schema b/apps/emqx_management/priv/emqx_management.schema index 343a70de6..a30a20e4d 100644 --- a/apps/emqx_management/priv/emqx_management.schema +++ b/apps/emqx_management/priv/emqx_management.schema @@ -227,6 +227,13 @@ end}. Prefix = "management.listener." ++ atom_to_list(Proto), case cuttlefish:conf_get(Prefix, Conf, undefined) of undefined -> Acc; + {IPStr, Port} -> + {ok, IP} = inet:parse_address(IPStr), + [{Proto, Port, [{ip, IP}] ++ TcpOpts(Prefix) ++ Opts(Prefix) + ++ case Proto of + http -> []; + https -> SslOpts(Prefix) + end} | Acc]; Port -> [{Proto, Port, TcpOpts(Prefix) ++ Opts(Prefix) ++ case Proto of @@ -236,4 +243,3 @@ end}. end end, [], [http, https]) end}. - diff --git a/apps/emqx_management/src/emqx_management.appup.src b/apps/emqx_management/src/emqx_management.appup.src index 9265913e6..15aef0463 100644 --- a/apps/emqx_management/src/emqx_management.appup.src +++ b/apps/emqx_management/src/emqx_management.appup.src @@ -1,13 +1,13 @@ %% -*- mode: erlang -*- {VSN, - [ {<<"4\\.3\\.([0-9]|1[0-2])">>, + [ {<<"4\\.3\\.([0-9]|1[0-4])">>, [ {apply,{minirest,stop_http,['http:management']}}, {apply,{minirest,stop_http,['https:management']}}, {restart_application, emqx_management} ]}, {<<".*">>, []} ], - [ {<<"4\\.3\\.([0-9]|1[0-2])">>, + [ {<<"4\\.3\\.([0-9]|1[0-4])">>, [ {apply,{minirest,stop_http,['http:management']}}, {apply,{minirest,stop_http,['https:management']}}, {restart_application, emqx_management} diff --git a/apps/emqx_management/src/emqx_mgmt_data_backup.erl b/apps/emqx_management/src/emqx_mgmt_data_backup.erl index 06dc0365a..cb94b7cf9 100644 --- a/apps/emqx_management/src/emqx_mgmt_data_backup.erl +++ b/apps/emqx_management/src/emqx_mgmt_data_backup.erl @@ -657,26 +657,38 @@ import(Filename, OverridesJson) -> -endif. do_import_data(Data, Version) -> - do_import_extra_data(Data, Version), import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version), import_blacklist(maps:get(<<"blacklist">>, Data, [])), import_applications(maps:get(<<"apps">>, Data, [])), import_users(maps:get(<<"users">>, Data, [])), + %% Import modules first to ensure the data of auth_mnesia module can be imported. + %% XXX: In opensource version, can't import if the emqx_auth_mnesia plug-in is not started?? + do_import_enterprise_modules(Data, Version), import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), import_auth_username(maps:get(<<"auth_username">>, Data, [])), import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), - import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])). + import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])), + %% always do extra import at last, to make sure resources are initiated before + %% creating the schemas + do_import_extra_data(Data, Version). -ifdef(EMQX_ENTERPRISE). do_import_extra_data(Data, _Version) -> _ = import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), - _ = import_modules(maps:get(<<"modules">>, Data, [])), _ = import_schemas(maps:get(<<"schemas">>, Data, [])), ok. -else. do_import_extra_data(_Data, _Version) -> ok. -endif. +-ifdef(EMQX_ENTERPRISE). +do_import_enterprise_modules(Data, _Version) -> + _ = import_modules(maps:get(<<"modules">>, Data, [])), + ok. +-else. +do_import_enterprise_modules(_Data, _Version) -> ok. +-endif. + covert_empty_headers([]) -> #{}; covert_empty_headers(Other) -> Other. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl new file mode 100644 index 000000000..021cf735a --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE.erl @@ -0,0 +1,73 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_auth_mnesia_data_export_import_SUITE). + +-compile([export_all, nowarn_export_all]). + +-ifdef(EMQX_ENTERPRISE). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx_modules/include/emqx_modules.hrl"). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Cfg) -> + _ = application:load(emqx_modules_spec), + emqx_ct_helpers:start_apps([emqx_rule_engine, emqx_modules, + emqx_management, emqx_dashboard]), + Cfg. + +end_per_suite(Cfg) -> + emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management, + emqx_modules, emqx_rule_engine]), + Cfg. + +get_data_path() -> + emqx_ct_helpers:deps_path(emqx_management, "test/emqx_auth_mnesia_data_export_import_SUITE_data/"). + +import(FilePath, _Version) -> + ok = emqx_mgmt_data_backup:import(get_data_path() ++ "/" ++ FilePath, <<"{}">>), + [_] = lists:filter( + fun(#module{type = mnesia_authentication}) -> true; + (_) -> false + end, emqx_modules_registry:get_modules()), + ?assertNotEqual(0, ets:info(emqx_user, size)), + ?assertNotEqual(0, ets:info(emqx_acl, size)). + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- + +t_importee427(_) -> + import("ee427.json", ee427), + {ok, _} = emqx_mgmt_data_backup:export(), + remove_all_users_and_acl(). + +t_importee430(_) -> + import("ee435.json", ee435), + {ok, _} = emqx_mgmt_data_backup:export(), + remove_all_users_and_acl(). + +remove_all_users_and_acl() -> + mnesia:delete_table(emqx_user), + mnesia:delete_table(emqx_acl). + +-endif. diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json new file mode 100644 index 000000000..43d58538a --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee427.json @@ -0,0 +1,88 @@ +{ + "version": "4.2", + "date": "2022-05-24 12:09:56", + "modules": [ + { + "id": "module:842a5c57", + "type": "mnesia_authentication", + "config": { + "password_hash": "sha256" + }, + "enabled": true, + "created_at": 1653365372585, + "description": "" + }, + { + "id": "module:e3d38e5a", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1652436434273, + "description": "" + }, + { + "id": "module:4f911150", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1652436434273, + "description": "" + }, + { + "id": "module:db11d08f", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1652436434273, + "description": "" + } + ], + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "oFu0ZiOAYJmB1DyOzoLwEervBK0=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "usera", + "type": "clientid", + "password": "WVlGsjBiNGJkOWRkN2QyZmYyOWViYjI4MzRiZjQyMDgzNDhkYzJjZmZlOGVjMjUzOGU5NDkwYmYyYjY5N2Q3NjUyMDU=", + "created_at": 1653365382492 + } + ], + "acl_mnesia": [ + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "pubsub", + "access": "allow", + "created_at": 1653365390351 + } + ], + "schemas": [], + "configs": [], + "listeners_state": [] +} diff --git a/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json new file mode 100644 index 000000000..e6d9cbb3b --- /dev/null +++ b/apps/emqx_management/test/emqx_auth_mnesia_data_export_import_SUITE_data/ee435.json @@ -0,0 +1,96 @@ +{ + "version": "4.3", + "rules": [], + "resources": [], + "blacklist": [], + "apps": [ + { + "id": "admin", + "secret": "public", + "name": "Default", + "desc": "Application user", + "status": true, + "expired": "undefined" + } + ], + "users": [ + { + "username": "admin", + "password": "02BzoSYaTxkscy2MDtU92EbX7b4=", + "tags": "administrator" + } + ], + "auth_mnesia": [ + { + "login": "usera", + "type": "clientid", + "password": "joYZ7GY2NzcxNTQwMzY4OTRjNWUyMTdmNDlkNmE5Yzc5MDJiNjA5OWRkMWRkZjc5N2E5OGI4YWFlYTdlOWNiMjU5OWE=", + "created_at": 1653360665243 + } + ], + "acl_mnesia": [ + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "pub", + "access": "allow", + "created_at": 1653360687955 + }, + { + "type": "clientid", + "type_value": "clientida", + "topic": "t/a", + "action": "sub", + "access": "allow", + "created_at": 1653360687955 + } + ], + "modules": [ + { + "id": "module:fcda7532", + "type": "mnesia_authentication", + "config": { + "password_hash": "sha256" + }, + "enabled": true, + "created_at": 1653360656060, + "description": "" + }, + { + "id": "module:db849123", + "type": "retainer", + "config": { + "storage_type": "ram", + "max_retained_messages": 0, + "max_payload_size": "1MB", + "expiry_interval": 0 + }, + "enabled": true, + "created_at": 1653360591111, + "description": "" + }, + { + "id": "module:55987aaa", + "type": "presence", + "config": { + "qos": 0 + }, + "enabled": true, + "created_at": 1653360591111, + "description": "" + }, + { + "id": "module:78cae4f9", + "type": "recon", + "config": {}, + "enabled": true, + "created_at": 1653360591111, + "description": "" + } + ], + "schemas": [], + "configs": [], + "listeners_state": [], + "date": "2022-05-24 10:51:39" +} diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index b96608edb..ded1bf46d 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -1,6 +1,6 @@ {application, emqx_prometheus, [{description, "Prometheus for EMQ X"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.1"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel,stdlib,prometheus]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.appup.src b/apps/emqx_prometheus/src/emqx_prometheus.appup.src new file mode 100644 index 000000000..a06d65fe1 --- /dev/null +++ b/apps/emqx_prometheus/src/emqx_prometheus.appup.src @@ -0,0 +1,9 @@ +%% -*- mode: erlang -*- +%% Unless you know what you are doing, DO NOT edit manually!! +{VSN, + [{"4.3.0", + [{load_module,emqx_prometheus,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}], + [{"4.3.0", + [{load_module,emqx_prometheus,brutal_purge,soft_purge,[]}]}, + {<<".*">>,[]}]}. diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 57c54446f..a7c450ba6 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -412,8 +412,8 @@ emqx_collect(emqx_client_connected, Stats) -> counter_metric(?C('client.connected', Stats)); emqx_collect(emqx_client_authenticate, Stats) -> counter_metric(?C('client.authenticate', Stats)); -emqx_collect(emqx_client_auth_anonymous, Stats) -> - counter_metric(?C('client.auth.anonymous', Stats)); +emqx_collect(emqx_client_auth_success_anonymous, Stats) -> + counter_metric(?C('client.auth.success.anonymous', Stats)); emqx_collect(emqx_client_check_acl, Stats) -> counter_metric(?C('client.check_acl', Stats)); emqx_collect(emqx_client_subscribe, Stats) -> @@ -566,7 +566,7 @@ emqx_metrics_delivery() -> emqx_metrics_client() -> [ emqx_client_connected , emqx_client_authenticate - , emqx_client_auth_anonymous + , emqx_client_auth_success_anonymous , emqx_client_check_acl , emqx_client_subscribe , emqx_client_unsubscribe diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 282c8929d..18e61967e 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -201,6 +201,8 @@ , unix_ts_to_rfc3339/2 , format_date/3 , format_date/4 + , timezone_to_second/1 + , date_to_unix_ts/3 , date_to_unix_ts/4 , rfc3339_to_unix_ts/1 , rfc3339_to_unix_ts/2 @@ -923,26 +925,37 @@ now_timestamp(Unit) -> time_unit(<<"second">>) -> second; time_unit(<<"millisecond">>) -> millisecond; time_unit(<<"microsecond">>) -> microsecond; -time_unit(<<"nanosecond">>) -> nanosecond. +time_unit(<<"nanosecond">>) -> nanosecond; +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond. format_date(TimeUnit, Offset, FormatString) -> - emqx_rule_utils:bin( - emqx_rule_date:date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString))). + Unit = time_unit(TimeUnit), + TimeEpoch = erlang:system_time(Unit), + format_date(Unit, Offset, FormatString, TimeEpoch). + +timezone_to_second(TimeZone) -> + emqx_calendar:offset_second(TimeZone). format_date(TimeUnit, Offset, FormatString, TimeEpoch) -> + Unit = time_unit(TimeUnit), emqx_rule_utils:bin( - emqx_rule_date:date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString), - TimeEpoch)). + lists:concat( + emqx_calendar:format(TimeEpoch, Unit, Offset, FormatString))). +%% date string has timezone information, calculate the offset. +date_to_unix_ts(TimeUnit, FormatString, InputString) -> + Unit = time_unit(TimeUnit), + emqx_calendar:parse(InputString, Unit, FormatString). + +%% date string has no timezone information, force add the offset. date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) -> - emqx_rule_date:parse_date(time_unit(TimeUnit), - emqx_rule_utils:str(Offset), - emqx_rule_utils:str(FormatString), - emqx_rule_utils:str(InputString)). + Unit = time_unit(TimeUnit), + OffsetSecond = emqx_calendar:offset_second(Offset), + OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit), + date_to_unix_ts(Unit, FormatString, InputString) - OffsetDelta. mongo_date() -> erlang:timestamp(). diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 98f6ad0b7..aa2781e8c 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -713,24 +713,29 @@ t_format_date_funcs(_) -> ?PROPTEST(prop_format_date_fun). prop_format_date_fun() -> - Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%y---%H:%M:%S%Z">>], + Args1 = [<<"second">>, <<"+07:00">>, <<"%m--%d--%Y---%H:%M:%S%z">>], ?FORALL(S, erlang:system_time(second), S == apply_func(date_to_unix_ts, Args1 ++ [apply_func(format_date, Args1 ++ [S])])), - Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%y---%H:%M:%S%Z">>], + Args2 = [<<"millisecond">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], + Args2DTUS = [<<"millisecond">>, <<"--%m--%d--%Y---%H:%M:%S:%3N%z">>], ?FORALL(S, erlang:system_time(millisecond), S == apply_func(date_to_unix_ts, - Args2 ++ [apply_func(format_date, + Args2DTUS ++ [apply_func(format_date, Args2 ++ [S])])), - Args = [<<"second">>, <<"+08:00">>, <<"%y-%m-%d-%H:%M:%S%Z">>], + Args = [<<"second">>, <<"+08:00">>, <<"%Y-%m-%d-%H:%M:%S%z">>], + ArgsDTUS = [<<"second">>, <<"%Y-%m-%d-%H:%M:%S%z">>], ?FORALL(S, erlang:system_time(second), S == apply_func(date_to_unix_ts, - Args ++ [apply_func(format_date, - Args ++ [S])])). -%%------------------------------------------------------------------------------ -%% Utility functions -%%------------------------------------------------------------------------------ + ArgsDTUS ++ [apply_func(format_date, + Args ++ [S])])), + % no offset in format string. force add offset + Second = erlang:system_time(second), + Args3 = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Second], + Formatters3 = apply_func(format_date, Args3), + Args3DTUS = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3], + Second == apply_func(date_to_unix_ts, Args3DTUS). apply_func(Name, Args) when is_atom(Name) -> erlang:apply(emqx_rule_funcs, Name, Args); diff --git a/bin/emqx b/bin/emqx index a3843ec7e..7eed9ab3c 100755 --- a/bin/emqx +++ b/bin/emqx @@ -37,15 +37,15 @@ assert_node_alive() { fi } -check_eralng_start() { - "$BINDIR/$PROGNAME" -noshell -boot "$REL_DIR/start_clean" -s crypto start -s init stop +check_erlang_start() { + "$BINDIR/$PROGNAME" -boot "$REL_DIR/start_clean" -eval "crypto:start(),halt()" } -if ! check_eralng_start >/dev/null 2>&1; then +if ! check_erlang_start >/dev/null 2>&1; then BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")" ## failed to start, might be due to missing libs, try to be portable export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH" - if ! check_eralng_start; then + if ! check_erlang_start; then ## it's hopeless echoerr "FATAL: Unable to start Erlang." echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed." @@ -338,14 +338,12 @@ generate_config() { ## changing the config 'log.rotation.size' rm -rf "${RUNNER_LOG_DIR}"/*.siz - EMQX_LICENSE_CONF_OPTION="" - if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then - EMQX_LICENSE_CONF_OPTION="-i ${EMQX_LICENSE_CONF}" - fi - set +e - # shellcheck disable=SC2086 - CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + if [ "${EMQX_LICENSE_CONF:-}" = "" ]; then + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + else + CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema -i "${EMQX_LICENSE_CONF}" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)" + fi # shellcheck disable=SC2181 RESULT=$? set -e diff --git a/bin/install_upgrade.escript b/bin/install_upgrade.escript index 69cec9175..14d90156c 100755 --- a/bin/install_upgrade.escript +++ b/bin/install_upgrade.escript @@ -5,6 +5,7 @@ -define(TIMEOUT, 300000). -define(INFO(Fmt,Args), io:format(Fmt++"~n",Args)). +-define(SEMVER_RE, <<"^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-[a-zA-Z\\d][-a-zA-Z.\\d]*)?(\\+[a-zA-Z\\d][-a-zA-Z.\\d]*)?$">>). -mode(compile). @@ -54,6 +55,7 @@ unpack(_, Args) -> install({RelName, NameTypeArg, NodeName, Cookie}, Opts) -> TargetNode = start_distribution(NodeName, NameTypeArg, Cookie), Version = proplists:get_value(version, Opts), + validate_target_version(Version, TargetNode), case unpack_release(RelName, TargetNode, Version) of {ok, Vsn} -> ?INFO("Unpacked successfully: ~p.", [Vsn]), @@ -427,6 +429,29 @@ erts_vsn() -> [ErtsVsn, _] = string:tokens(binary_to_list(Str), " "), ErtsVsn. +validate_target_version(TargetVersion, TargetNode) -> + CurrentVersion = current_release_version(TargetNode), + case {get_major_minor_vsn(CurrentVersion), get_major_minor_vsn(TargetVersion)} of + {{Major, Minor}, {Major, Minor}} -> ok; + _ -> + ?INFO("Cannot upgrade/downgrade to ~s from ~s~n" + "We only support relup between patch versions", + [TargetVersion, CurrentVersion]), + error({relup_not_allowed, unsupported_target_version}) + end. + +get_major_minor_vsn(Version) -> + Parts = parse_semver(Version), + [Major | Rem0] = Parts, + [Minor | _Rem1] = Rem0, + {Major, Minor}. + +parse_semver(Version) -> + case re:run(Version, ?SEMVER_RE, [{capture, all_but_first, binary}]) of + {match, Parts} -> Parts; + nomatch -> error({invalid_semver, Version}) + end. + str(A) when is_atom(A) -> atom_to_list(A); str(A) when is_binary(A) -> diff --git a/bin/node_dump b/bin/node_dump index 83b41332b..79b469112 100755 --- a/bin/node_dump +++ b/bin/node_dump @@ -1,13 +1,13 @@ #!/bin/sh set -eu -ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" -echo "Running node dump in ${ROOT_DIR}" +RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)" +echo "Running node dump in ${RUNNER_ROOT_DIR}" # shellcheck disable=SC1090 -. "$ROOT_DIR"/releases/emqx_vars +. "$RUNNER_ROOT_DIR"/releases/emqx_vars -cd "${ROOT_DIR}" +cd "${RUNNER_ROOT_DIR}" DUMP="$RUNNER_LOG_DIR/node_dump_$(date +"%Y%m%d_%H%M%S").tar.gz" CONF_DUMP="$RUNNER_LOG_DIR/conf.dump" diff --git a/etc/emqx.conf b/etc/emqx.conf index bc45df1fc..0f8f93a91 100644 --- a/etc/emqx.conf +++ b/etc/emqx.conf @@ -458,7 +458,33 @@ log.file = emqx.log ## Log formatter ## Value: text | json -#log.formatter = text +log.formatter = text + +## Format of the text logger. +## +## Value: rfc3339 | FORMAT +## Where FORMAT is the format string of the timestamp. Supported specifiers: +## %Y: year +## %m: month +## %d: day +## %H: hour +## %M: minute +## %S: second +## %N: nanoseconds (000000000 - 999999999) +## %6N: microseconds (00000 - 999999) +## %3N: milliseconds (000 - 999) +## %z: timezone, [+-]HHMM +## %:z: timezone, [+-]HH:MM +## %::z: timezone, [+-]HH:MM:SS +## +## For example: +## log.formatter.text.date.format = %Y-%m-%dT%H:%M:%S.%6N %:z +## +## Before 4.2, the default date format was: +## log.formatter.text.date.format = %Y-%m-%d %H:%M:%S.%3N +## +## Default: rfc3339 +# log.formatter.text.date.format = rfc3339 ## Log to single line ## Value: Boolean diff --git a/include/emqx_release.hrl b/include/emqx_release.hrl index 5145f7b5e..43d9dfcb7 100644 --- a/include/emqx_release.hrl +++ b/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "4.3.14"}). +-define(EMQX_RELEASE, {opensource, "4.3.15-rc.2"}). -else. diff --git a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema index d517a6e97..a2985429b 100644 --- a/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema +++ b/lib-ce/emqx_dashboard/priv/emqx_dashboard.schema @@ -83,7 +83,7 @@ ]}. {mapping, "dashboard.listener.https.verify", "emqx_dashboard.listeners", [ - {datatype, string} + {datatype, atom} ]}. {mapping, "dashboard.listener.https.fail_if_no_peer_cert", "emqx_dashboard.listeners", [ @@ -149,4 +149,3 @@ end end, [http, https])) end}. - diff --git a/priv/emqx.schema b/priv/emqx.schema index 4d7b1beda..304fc0634 100644 --- a/priv/emqx.schema +++ b/priv/emqx.schema @@ -528,6 +528,12 @@ end}. {datatype, {enum, [text, json]}} ]}. +%% @doc format logs as text, date format part +{mapping, "log.formatter.text.date.format", "kernel.logger", [ + {default, "rfc3339"}, + {datatype, string} +]}. + %% @doc format logs in a single line. {mapping, "log.single_line", "kernel.logger", [ {default, true}, @@ -629,9 +635,38 @@ end}. single_line => SingleLine }}; text -> + DateFormat = + case cuttlefish:conf_get("log.formatter.text.date.format", Conf, "rfc3339") of + "rfc3339" -> + rfc3339; + DateStr -> + DateStrTrans = + fun + DST(<<>>, Formatter) -> lists:reverse(Formatter); + DST(<<"%Y", Tail/binary>>, Formatter) -> DST(Tail, [year | Formatter]); + DST(<<"%m", Tail/binary>>, Formatter) -> DST(Tail, [month | Formatter]); + DST(<<"%d", Tail/binary>>, Formatter) -> DST(Tail, [day | Formatter]); + DST(<<"%H", Tail/binary>>, Formatter) -> DST(Tail, [hour | Formatter]); + DST(<<"%M", Tail/binary>>, Formatter) -> DST(Tail, [minute | Formatter]); + DST(<<"%S", Tail/binary>>, Formatter) -> DST(Tail, [second | Formatter]); + DST(<<"%N", Tail/binary>>, Formatter) -> DST(Tail, [nanosecond | Formatter]); + DST(<<"%3N", Tail/binary>>, Formatter) -> DST(Tail, [millisecond | Formatter]); + DST(<<"%6N", Tail/binary>>, Formatter) -> DST(Tail, [microsecond | Formatter]); + DST(<<"%z", Tail/binary>>, Formatter) -> DST(Tail, [timezone | Formatter]); + DST(<<"%:z", Tail/binary>>, Formatter) -> DST(Tail, [timezone1 | Formatter]); + DST(<<"%::z", Tail/binary>>, Formatter) -> DST(Tail, [timezone2 | Formatter]); + DST(<>, [Str | Formatter]) when is_list(Str) -> + DST(Tail, [lists:append(Str, [Char]) | Formatter]); + DST(<>, Formatter) -> + DST(Tail, [[Char] | Formatter]) + end, + DateStrTrans(list_to_binary(DateStr), []) + end, {emqx_logger_textfmt, - #{template => - [time," [",level,"] ", + #{ + date_format => DateFormat, + template => + [" [",level,"] ", {clientid, [{peername, [clientid,"@",peername," "], diff --git a/rebar.config b/rebar.config index 8ed5933fc..2307f3620 100644 --- a/rebar.config +++ b/rebar.config @@ -58,8 +58,9 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.15.0"}}} - , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.2.1"}}} + , {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.1"}}} , {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.13"}}} + , {epgsql, {git, "https://github.com/emqx/epgsql.git", {tag, "4.6.0"}}} ]}. {xref_ignores, diff --git a/scripts/get-dashboard.sh b/scripts/get-dashboard.sh index abe8e2415..62467420c 100755 --- a/scripts/get-dashboard.sh +++ b/scripts/get-dashboard.sh @@ -9,7 +9,7 @@ PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" case "${PKG_VSN}" in 4.3*) EMQX_CE_DASHBOARD_VERSION='v4.3.7' - EMQX_EE_DASHBOARD_VERSION='v4.3.18' + EMQX_EE_DASHBOARD_VERSION='v4.3.19' ;; *) echo "Unsupported version $PKG_VSN" >&2 diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 840f63509..dd8dd9ca2 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -168,8 +168,8 @@ find_appup_actions(App, OldDowngrade = ensure_all_patch_versions(App, CurrVersion, OldDowngrade0), UpDiff = diff_app(up, App, CurrAppIdx, PrevAppIdx), DownDiff = diff_app(down, App, PrevAppIdx, CurrAppIdx), - Upgrade = merge_update_actions(App, UpDiff, OldUpgrade), - Downgrade = merge_update_actions(App, DownDiff, OldDowngrade), + Upgrade = merge_update_actions(App, UpDiff, OldUpgrade, PrevVersion), + Downgrade = merge_update_actions(App, DownDiff, OldDowngrade, PrevVersion), case OldUpgrade =:= Upgrade andalso OldDowngrade =:= Downgrade of true -> []; false -> [{App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}] @@ -258,14 +258,40 @@ find_base_appup_actions(App, PrevVersion) -> end, {ensure_version(PrevVersion, Upgrade), ensure_version(PrevVersion, Downgrade)}. -merge_update_actions(App, Changes, Vsns) -> +merge_update_actions(App, Changes, Vsns, PrevVersion) -> lists:map(fun(Ret = {<<".*">>, _}) -> Ret; ({Vsn, Actions}) -> - {Vsn, do_merge_update_actions(App, Changes, Actions)} + case is_skipped_version(App, Vsn, PrevVersion) of + true -> + log("WARN: ~p has version ~s skipped over?~n", [App, Vsn]), + {Vsn, Actions}; + false -> + {Vsn, do_merge_update_actions(App, Changes, Actions)} + end end, Vsns). +%% say current version is 1.1.3, and the compare base is version 1.1.1, +%% but there is a 1.1.2 in appup we may skip merging instructions for +%% 1.1.2 because it's not used and no way to know what has been changed +is_skipped_version(App, Vsn, PrevVersion) when is_list(Vsn) andalso is_list(PrevVersion) -> + case is_app_external(App) andalso parse_version_number(Vsn) of + {ok, VsnTuple} -> + case parse_version_number(PrevVersion) of + {ok, PrevVsnTuple} -> + VsnTuple > PrevVsnTuple; + _ -> + false + end; + _ -> + false + end; +is_skipped_version(_App, _Vsn, _PrevVersion) -> + %% if app version is a regexp, we don't know for sure + %% return 'false' to be on the safe side + false. + do_merge_update_actions(App, {New0, Changed0, Deleted0}, OldActions) -> AppSpecific = app_specific_actions(App) -- OldActions, AlreadyHandled = lists:flatten(lists:map(fun process_old_action/1, OldActions)), diff --git a/src/emqx.appup.src b/src/emqx.appup.src index 09ad2d13a..9dbf47f6f 100644 --- a/src/emqx.appup.src +++ b/src/emqx.appup.src @@ -2,15 +2,25 @@ %% Unless you know what you are doing, DO NOT edit manually!! {VSN, [{"4.3.15", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, @@ -20,9 +30,17 @@ {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -32,16 +50,23 @@ {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_misc,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -56,18 +81,23 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -79,6 +109,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -88,14 +119,17 @@ {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -107,6 +141,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, @@ -117,13 +152,16 @@ {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -134,6 +172,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -150,12 +189,15 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -166,6 +208,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -181,13 +224,15 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -197,6 +242,7 @@ {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -214,13 +260,15 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -229,6 +277,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -247,13 +296,15 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -261,6 +312,7 @@ {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -280,19 +332,22 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -313,19 +368,22 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -347,19 +405,22 @@ {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -382,17 +443,19 @@ {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_vm,brutal_purge,soft_purge,[]}, @@ -418,18 +481,20 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{add_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_vm_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {apply,{emqx_metrics,assign_auth_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,assign_acl_stats_from_ets_to_counter,[]}}, {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, @@ -458,34 +523,52 @@ {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, - {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, + {update,emqx_os_mon,{advanced,[]}}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], [{"4.3.15", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}]}, {"4.3.14", - [{load_module,emqx_misc,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_misc,brutal_purge,soft_purge,[]}, {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, + {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}]}, {"4.3.13", - [{load_module,emqx_session,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_session,brutal_purge,soft_purge,[]}, {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, @@ -498,13 +581,18 @@ {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, + {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_control,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_ctl,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, {"4.3.12", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -529,7 +617,11 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.11", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -548,7 +640,6 @@ {load_module,emqx_vm,brutal_purge,soft_purge,[]}, {load_module,emqx_sys_mon,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, @@ -556,7 +647,11 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.10", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -576,14 +671,17 @@ {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.9", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -607,14 +705,17 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.8", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, + {load_module,emqx_alarm_handler,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -638,14 +739,15 @@ {load_module,emqx_mqueue,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_rpc,brutal_purge,soft_purge,[]}, - {load_module,emqx_alarm,brutal_purge,soft_purge,[]}, {load_module,emqx_os_mon,brutal_purge,soft_purge,[]}, {load_module,emqx,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.7", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -677,7 +779,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.6", - [{load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_access_rule,brutal_purge,soft_purge,[]}, {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, @@ -709,7 +813,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.5", - [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, @@ -741,7 +847,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.4", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -773,7 +881,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.3", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -806,7 +916,9 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.2", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_plugins,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, @@ -839,7 +951,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.1", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, @@ -875,7 +988,8 @@ {load_module,emqx_message,brutal_purge,soft_purge,[]}, {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_hooks,brutal_purge,soft_purge,[]}, + [{delete_module,emqx_calendar}, + {load_module,emqx_hooks,brutal_purge,soft_purge,[]}, {load_module,emqx_pmon,brutal_purge,soft_purge,[]}, {load_module,emqx_banned,brutal_purge,soft_purge,[]}, {load_module,emqx_sys,brutal_purge,soft_purge,[]}, diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl index fe12ce7e6..93dde28c2 100644 --- a/src/emqx_access_control.erl +++ b/src/emqx_access_control.erl @@ -34,11 +34,15 @@ -spec(authenticate(emqx_types:clientinfo()) -> {ok, result()} | {error, term()}). authenticate(ClientInfo = #{zone := Zone}) -> AuthResult = default_auth_result(Zone), - case emqx_zone:get_env(Zone, bypass_auth_plugins, false) of + case + begin ok = emqx_metrics:inc('client.authenticate'), + emqx_zone:get_env(Zone, bypass_auth_plugins, false) + end + of true -> return_auth_result(AuthResult); false -> - return_auth_result(run_hooks('client.authenticate', [ClientInfo], AuthResult)) + return_auth_result(emqx_hooks:run_fold('client.authenticate', [ClientInfo], AuthResult)) end. %% @doc Check ACL @@ -51,6 +55,10 @@ check_acl(ClientInfo, PubSub, Topic) -> end, inc_acl_metrics(Result), Result. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +%% ACL check_acl_cache(ClientInfo, PubSub, Topic) -> case emqx_acl_cache:get_acl_cache(PubSub, Topic) of not_found -> @@ -64,21 +72,16 @@ check_acl_cache(ClientInfo, PubSub, Topic) -> do_check_acl(ClientInfo = #{zone := Zone}, PubSub, Topic) -> Default = emqx_zone:get_env(Zone, acl_nomatch, deny), - case run_hooks('client.check_acl', [ClientInfo, PubSub, Topic], Default) of + case + begin + ok = emqx_metrics:inc('client.check_acl'), + emqx_hooks:run_fold('client.check_acl', [ClientInfo, PubSub, Topic], Default) + end + of allow -> allow; _Other -> deny end. -default_auth_result(Zone) -> - case emqx_zone:get_env(Zone, allow_anonymous, false) of - true -> #{auth_result => success, anonymous => true}; - false -> #{auth_result => not_authorized, anonymous => false} - end. - --compile({inline, [run_hooks/3]}). -run_hooks(Name, Args, Acc) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). - -compile({inline, [inc_acl_metrics/1]}). inc_acl_metrics(allow) -> emqx_metrics:inc('client.acl.allow'); @@ -87,8 +90,26 @@ inc_acl_metrics(deny) -> inc_acl_metrics(cache_hit) -> emqx_metrics:inc('client.acl.cache_hit'). +%% Auth +default_auth_result(Zone) -> + case emqx_zone:get_env(Zone, allow_anonymous, false) of + true -> #{auth_result => success, anonymous => true}; + false -> #{auth_result => not_authorized, anonymous => false} + end. + -compile({inline, [return_auth_result/1]}). -return_auth_result(Result = #{auth_result := success}) -> - {ok, Result}; -return_auth_result(Result) -> - {error, maps:get(auth_result, Result, unknown_error)}. +return_auth_result(AuthResult = #{auth_result := success}) -> + inc_auth_success_metrics(AuthResult), + {ok, AuthResult}; +return_auth_result(AuthResult) -> + emqx_metrics:inc('client.auth.failure'), + {error, maps:get(auth_result, AuthResult, unknown_error)}. + +-compile({inline, [inc_auth_success_metrics/1]}). +inc_auth_success_metrics(AuthResult) -> + is_anonymous(AuthResult) andalso + emqx_metrics:inc('client.auth.success.anonymous'), + emqx_metrics:inc('client.auth.success'). + +is_anonymous(#{anonymous := true}) -> true; +is_anonymous(_AuthResult) -> false. diff --git a/src/emqx_alarm.erl b/src/emqx_alarm.erl index 6573ec1b5..e3b45e295 100644 --- a/src/emqx_alarm.erl +++ b/src/emqx_alarm.erl @@ -39,6 +39,8 @@ , deactivate/1 , deactivate/2 , delete_all_deactivated_alarms/0 + , ensure_deactivated/1 + , ensure_deactivated/2 , get_alarms/0 , get_alarms/1 ]). @@ -132,6 +134,24 @@ activate(Name) -> activate(Name, Details) -> gen_server:call(?MODULE, {activate_alarm, Name, Details}). +-spec ensure_deactivated(binary() | atom()) -> ok. +ensure_deactivated(Name) -> + ensure_deactivated(Name, no_details). + +-spec ensure_deactivated(binary() | atom(), atom() | map()) -> ok. +ensure_deactivated(Name, Data) -> + %% this duplicates the dirty read in handle_call, + %% intention is to avoid making gen_server calls when there is no alarm + case mnesia:dirty_read(?ACTIVATED_ALARM, Name) of + [] -> + ok; + _ -> + case deactivate(Name, Data) of + {error, not_found} -> ok; + Other -> Other + end + end. + deactivate(Name) -> gen_server:call(?MODULE, {deactivate_alarm, Name, no_details}). diff --git a/src/emqx_alarm_handler.erl b/src/emqx_alarm_handler.erl index b389d4175..1520a231b 100644 --- a/src/emqx_alarm_handler.erl +++ b/src/emqx_alarm_handler.erl @@ -56,21 +56,12 @@ init({_Args, {alarm_handler, _ExistingAlarms}}) -> init(_) -> {ok, []}. -handle_event({set_alarm, {system_memory_high_watermark, []}}, State) -> - emqx_alarm:activate(high_system_memory_usage, #{high_watermark => emqx_os_mon:get_sysmem_high_watermark()}), - {ok, State}; - -handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) -> +handle_event({set_alarm, {process_memory_high_watermark, Pid}}, State) -> emqx_alarm:activate(high_process_memory_usage, #{pid => list_to_binary(pid_to_list(Pid)), high_watermark => emqx_os_mon:get_procmem_high_watermark()}), {ok, State}; - -handle_event({clear_alarm, system_memory_high_watermark}, State) -> - emqx_alarm:deactivate(high_system_memory_usage), - {ok, State}; - -handle_event({clear_alarm, process_memory_high_watermark}, State) -> - emqx_alarm:deactivate(high_process_memory_usage), +handle_event({clear_alarm, process_memory_high_watermark}, State) -> + emqx_alarm:ensure_deactivate(high_process_memory_usage), {ok, State}; handle_event(_, State) -> diff --git a/src/emqx_calendar.erl b/src/emqx_calendar.erl new file mode 100644 index 000000000..0c094df81 --- /dev/null +++ b/src/emqx_calendar.erl @@ -0,0 +1,436 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2019-2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_calendar). + +-define(SECONDS_PER_MINUTE, 60). +-define(SECONDS_PER_HOUR, 3600). +-define(SECONDS_PER_DAY, 86400). +-define(DAYS_PER_YEAR, 365). +-define(DAYS_PER_LEAP_YEAR, 366). +-define(DAYS_FROM_0_TO_1970, 719528). +-define(SECONDS_FROM_0_TO_1970, ?DAYS_FROM_0_TO_1970 * ?SECONDS_PER_DAY). + +-export([ formatter/1 + , format/3 + , format/4 + , parse/3 + , offset_second/1 + ]). + +-define(DATE_PART, + [ year + , month + , day + , hour + , minute + , second + , nanosecond + , millisecond + , microsecond + ]). + +-define(DATE_ZONE_NAME, + [ timezone + , timezone1 + , timezone2 + ]). + +formatter(FormatterStr) when is_list(FormatterStr) -> + formatter(list_to_binary(FormatterStr)); +formatter(FormatterBin) when is_binary(FormatterBin) -> + do_formatter(FormatterBin, []). + +offset_second(Offset) -> + offset_second_(Offset). + +format(Time, Unit, Formatter) -> + format(Time, Unit, undefined, Formatter). + +format(Time, Unit, Offset, FormatterBin) when is_binary(FormatterBin) -> + format(Time, Unit, Offset, formatter(FormatterBin)); +format(Time, Unit, Offset, Formatter) -> + do_format(Time, time_unit(Unit), offset_second(Offset), Formatter). + +parse(DateStr, Unit, FormatterBin) when is_binary(FormatterBin) -> + parse(DateStr, Unit, formatter(FormatterBin)); +parse(DateStr, Unit, Formatter) -> + do_parse(DateStr, Unit, Formatter). +%% ------------------------------------------------------------------------------------------------- +%% internal + +time_unit(second) -> second; +time_unit(millisecond) -> millisecond; +time_unit(microsecond) -> microsecond; +time_unit(nanosecond) -> nanosecond; +time_unit("second") -> second; +time_unit("millisecond") -> millisecond; +time_unit("microsecond") -> microsecond; +time_unit("nanosecond") -> nanosecond; +time_unit(<<"second">>) -> second; +time_unit(<<"millisecond">>) -> millisecond; +time_unit(<<"microsecond">>) -> microsecond; +time_unit(<<"nanosecond">>) -> nanosecond. + +%% ------------------------------------------------------------------------------------------------- +%% internal: format part + +do_formatter(<<>>, Formatter) -> lists:reverse(Formatter); +do_formatter(<<"%Y", Tail/binary>>, Formatter) -> do_formatter(Tail, [year | Formatter]); +do_formatter(<<"%m", Tail/binary>>, Formatter) -> do_formatter(Tail, [month | Formatter]); +do_formatter(<<"%d", Tail/binary>>, Formatter) -> do_formatter(Tail, [day | Formatter]); +do_formatter(<<"%H", Tail/binary>>, Formatter) -> do_formatter(Tail, [hour | Formatter]); +do_formatter(<<"%M", Tail/binary>>, Formatter) -> do_formatter(Tail, [minute | Formatter]); +do_formatter(<<"%S", Tail/binary>>, Formatter) -> do_formatter(Tail, [second | Formatter]); +do_formatter(<<"%N", Tail/binary>>, Formatter) -> do_formatter(Tail, [nanosecond | Formatter]); +do_formatter(<<"%3N", Tail/binary>>, Formatter) -> do_formatter(Tail, [millisecond | Formatter]); +do_formatter(<<"%6N", Tail/binary>>, Formatter) -> do_formatter(Tail, [microsecond | Formatter]); +do_formatter(<<"%z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone | Formatter]); +do_formatter(<<"%:z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone1 | Formatter]); +do_formatter(<<"%::z", Tail/binary>>, Formatter) -> do_formatter(Tail, [timezone2 | Formatter]); +do_formatter(<>, [Str | Formatter]) when is_list(Str) -> + do_formatter(Tail, [lists:append(Str, [Char]) | Formatter]); +do_formatter(<>, Formatter) -> do_formatter(Tail, [[Char] | Formatter]). + +offset_second_(OffsetSecond) when is_integer(OffsetSecond) -> OffsetSecond; +offset_second_(undefined) -> 0; +offset_second_("local") -> offset_second_(local); +offset_second_(<<"local">>) -> offset_second_(local); +offset_second_(local) -> + UniversalTime = calendar:system_time_to_universal_time(erlang:system_time(second), second), + LocalTime = erlang:universaltime_to_localtime(UniversalTime), + LocalSecs = calendar:datetime_to_gregorian_seconds(LocalTime), + UniversalSecs = calendar:datetime_to_gregorian_seconds(UniversalTime), + LocalSecs - UniversalSecs; +offset_second_(Offset) when is_binary(Offset) -> + offset_second_(erlang:binary_to_list(Offset)); +offset_second_("Z") -> 0; +offset_second_("z") -> 0; +offset_second_(Offset) when is_list(Offset) -> + Sign = hd(Offset), + ((Sign == $+) orelse (Sign == $-)) + orelse error({bad_time_offset, Offset}), + Signs = #{$+ => 1, $- => -1}, + PosNeg = maps:get(Sign, Signs), + [Sign | HM] = Offset, + {HourStr, MinuteStr, SecondStr} = + case string:tokens(HM, ":") of + [H, M] -> + {H, M, "0"}; + [H, M, S] -> + {H, M, S}; + [HHMM] when erlang:length(HHMM) == 4 -> + {string:sub_string(HHMM, 1,2), string:sub_string(HHMM, 3,4), "0"}; + _ -> + error({bad_time_offset, Offset}) + end, + Hour = erlang:list_to_integer(HourStr), + Minute = erlang:list_to_integer(MinuteStr), + Second = erlang:list_to_integer(SecondStr), + (Hour =< 23) orelse error({bad_time_offset_hour, Hour}), + (Minute =< 59) orelse error({bad_time_offset_minute, Minute}), + (Second =< 59) orelse error({bad_time_offset_second, Second}), + PosNeg * (Hour * 3600 + Minute * 60 + Second). + +do_format(Time, Unit, Offset, Formatter) -> + Adjustment = erlang:convert_time_unit(Offset, second, Unit), + AdjustedTime = Time + Adjustment, + Factor = factor(Unit), + Secs = AdjustedTime div Factor, + DateTime = system_time_to_datetime(Secs), + {{Year, Month, Day}, {Hour, Min, Sec}} = DateTime, + Date = #{ + year => padding(Year, 4), + month => padding(Month, 2), + day => padding(Day, 2), + hour => padding(Hour, 2), + minute => padding(Min, 2), + second => padding(Sec, 2), + millisecond => trans_x_second(Unit, millisecond, Time), + microsecond => trans_x_second(Unit, microsecond, Time), + nanosecond => trans_x_second(Unit, nanosecond, Time) + }, + Timezones = formatter_timezones(Offset, Formatter, #{}), + DateWithZone = maps:merge(Date, Timezones), + [maps:get(Key, DateWithZone, Key) || Key <- Formatter]. + +formatter_timezones(_Offset, [], Zones) -> Zones; +formatter_timezones(Offset, [Timezone | Formatter], Zones) -> + case lists:member(Timezone, [timezone, timezone1, timezone2]) of + true -> + NZones = Zones#{Timezone => offset_to_timezone(Offset, Timezone)}, + formatter_timezones(Offset, Formatter, NZones); + false -> + formatter_timezones(Offset, Formatter, Zones) + end. + +offset_to_timezone(Offset, Timezone) -> + Sign = + case Offset >= 0 of + true -> + $+; + false -> + $- + end, + {H, M, S} = seconds_to_time(abs(Offset)), + %% TODO: Support zone define %:::z + %% Numeric time zone with ":" to necessary precision (e.g., -04, +05:30). + case Timezone of + timezone -> + %% +0800 + io_lib:format("~c~2.10.0B~2.10.0B", [Sign, H, M]); + timezone1 -> + %% +08:00 + io_lib:format("~c~2.10.0B:~2.10.0B", [Sign, H, M]); + timezone2 -> + %% +08:00:00 + io_lib:format("~c~2.10.0B:~2.10.0B:~2.10.0B", [Sign, H, M, S]) + end. + +factor(second) -> 1; +factor(millisecond) -> 1000; +factor(microsecond) -> 1000000; +factor(nanosecond) -> 1000000000. + +system_time_to_datetime(Seconds) -> + gregorian_seconds_to_datetime(Seconds + ?SECONDS_FROM_0_TO_1970). + +gregorian_seconds_to_datetime(Secs) when Secs >= 0 -> + Days = Secs div ?SECONDS_PER_DAY, + Rest = Secs rem ?SECONDS_PER_DAY, + {gregorian_days_to_date(Days), seconds_to_time(Rest)}. + +seconds_to_time(Secs) when Secs >= 0, Secs < ?SECONDS_PER_DAY -> + Secs0 = Secs rem ?SECONDS_PER_DAY, + Hour = Secs0 div ?SECONDS_PER_HOUR, + Secs1 = Secs0 rem ?SECONDS_PER_HOUR, + Minute = Secs1 div ?SECONDS_PER_MINUTE, + Second = Secs1 rem ?SECONDS_PER_MINUTE, + {Hour, Minute, Second}. + +gregorian_days_to_date(Days) -> + {Year, DayOfYear} = day_to_year(Days), + {Month, DayOfMonth} = year_day_to_date(Year, DayOfYear), + {Year, Month, DayOfMonth}. + +day_to_year(DayOfEpoch) when DayOfEpoch >= 0 -> + YMax = DayOfEpoch div ?DAYS_PER_YEAR, + YMin = DayOfEpoch div ?DAYS_PER_LEAP_YEAR, + {Y1, D1} = dty(YMin, YMax, DayOfEpoch, dy(YMin), dy(YMax)), + {Y1, DayOfEpoch - D1}. + +year_day_to_date(Year, DayOfYear) -> + ExtraDay = + case is_leap_year(Year) of + true -> + 1; + false -> + 0 + end, + {Month, Day} = year_day_to_date2(ExtraDay, DayOfYear), + {Month, Day + 1}. + +dty(Min, Max, _D1, DMin, _DMax) when Min == Max -> + {Min, DMin}; +dty(Min, Max, D1, DMin, DMax) -> + Diff = Max - Min, + Mid = Min + Diff * (D1 - DMin) div (DMax - DMin), + MidLength = + case is_leap_year(Mid) of + true -> + ?DAYS_PER_LEAP_YEAR; + false -> + ?DAYS_PER_YEAR + end, + case dy(Mid) of + D2 when D1 < D2 -> + NewMax = Mid - 1, + dty(Min, NewMax, D1, DMin, dy(NewMax)); + D2 when D1 - D2 >= MidLength -> + NewMin = Mid + 1, + dty(NewMin, Max, D1, dy(NewMin), DMax); + D2 -> + {Mid, D2} + end. + +dy(Y) when Y =< 0 -> + 0; +dy(Y) -> + X = Y - 1, + X div 4 - X div 100 + X div 400 + X * ?DAYS_PER_YEAR + ?DAYS_PER_LEAP_YEAR. + +is_leap_year(Y) when is_integer(Y), Y >= 0 -> + is_leap_year1(Y). + +is_leap_year1(Year) when Year rem 4 =:= 0, Year rem 100 > 0 -> + true; +is_leap_year1(Year) when Year rem 400 =:= 0 -> + true; +is_leap_year1(_) -> + false. + +year_day_to_date2(_, Day) when Day < 31 -> + {1, Day}; +year_day_to_date2(E, Day) when 31 =< Day, Day < 59 + E -> + {2, Day - 31}; +year_day_to_date2(E, Day) when 59 + E =< Day, Day < 90 + E -> + {3, Day - (59 + E)}; +year_day_to_date2(E, Day) when 90 + E =< Day, Day < 120 + E -> + {4, Day - (90 + E)}; +year_day_to_date2(E, Day) when 120 + E =< Day, Day < 151 + E -> + {5, Day - (120 + E)}; +year_day_to_date2(E, Day) when 151 + E =< Day, Day < 181 + E -> + {6, Day - (151 + E)}; +year_day_to_date2(E, Day) when 181 + E =< Day, Day < 212 + E -> + {7, Day - (181 + E)}; +year_day_to_date2(E, Day) when 212 + E =< Day, Day < 243 + E -> + {8, Day - (212 + E)}; +year_day_to_date2(E, Day) when 243 + E =< Day, Day < 273 + E -> + {9, Day - (243 + E)}; +year_day_to_date2(E, Day) when 273 + E =< Day, Day < 304 + E -> + {10, Day - (273 + E)}; +year_day_to_date2(E, Day) when 304 + E =< Day, Day < 334 + E -> + {11, Day - (304 + E)}; +year_day_to_date2(E, Day) when 334 + E =< Day -> + {12, Day - (334 + E)}. + +trans_x_second(FromUnit, ToUnit, Time) -> + XSecond = do_trans_x_second(FromUnit, ToUnit, Time), + Len = + case ToUnit of + millisecond -> 3; + microsecond -> 6; + nanosecond -> 9 + end, + padding(XSecond, Len). + +do_trans_x_second(second, second, Time) -> Time div 60; +do_trans_x_second(second, _, _Time) -> 0; + +do_trans_x_second(millisecond, millisecond, Time) -> Time rem 1000; +do_trans_x_second(millisecond, microsecond, Time) -> (Time rem 1000) * 1000; +do_trans_x_second(millisecond, nanosecond, Time) -> (Time rem 1000) * 1000_000; + +do_trans_x_second(microsecond, millisecond, Time) -> Time div 1000 rem 1000; +do_trans_x_second(microsecond, microsecond, Time) -> Time rem 1000000; +do_trans_x_second(microsecond, nanosecond, Time) -> (Time rem 1000000) * 1000; + +do_trans_x_second(nanosecond, millisecond, Time) -> Time div 1000000 rem 1000; +do_trans_x_second(nanosecond, microsecond, Time) -> Time div 1000 rem 1000000; +do_trans_x_second(nanosecond, nanosecond, Time) -> Time rem 1000000000. + +padding(Data, Len) when is_integer(Data) -> + padding(integer_to_list(Data), Len); +padding(Data, Len) when Len > 0 andalso erlang:length(Data) < Len -> + [$0 | padding(Data, Len - 1)]; +padding(Data, _Len) -> + Data. + +%% ------------------------------------------------------------------------------------------------- +%% internal +%% parse part + +do_parse(DateStr, Unit, Formatter) -> + DateInfo = do_parse_date_str(DateStr, Formatter, #{}), + {Precise, PrecisionUnit} = precision(DateInfo), + Counter = + fun + (year, V, Res) -> + Res + dy(V) * ?SECONDS_PER_DAY * Precise - (?SECONDS_FROM_0_TO_1970 * Precise); + (month, V, Res) -> + Res + dm(V) * ?SECONDS_PER_DAY * Precise; + (day, V, Res) -> + Res + (V * ?SECONDS_PER_DAY * Precise); + (hour, V, Res) -> + Res + (V * ?SECONDS_PER_HOUR * Precise); + (minute, V, Res) -> + Res + (V * ?SECONDS_PER_MINUTE * Precise); + (second, V, Res) -> + Res + V * Precise; + (millisecond, V, Res) -> + case PrecisionUnit of + millisecond -> + Res + V; + microsecond -> + Res + (V * 1000); + nanosecond -> + Res + (V * 1000000) + end; + (microsecond, V, Res) -> + case PrecisionUnit of + microsecond -> + Res + V; + nanosecond -> + Res + (V * 1000) + end; + (nanosecond, V, Res) -> + Res + V; + (parsed_offset, V, Res) -> + Res - V + end, + Count = maps:fold(Counter, 0, DateInfo) - (?SECONDS_PER_DAY * Precise), + erlang:convert_time_unit(Count, PrecisionUnit, Unit). + +precision(#{nanosecond := _}) -> {1000_000_000, nanosecond}; +precision(#{microsecond := _}) -> {1000_000, microsecond}; +precision(#{millisecond := _}) -> {1000, millisecond}; +precision(#{second := _}) -> {1, second}; +precision(_) -> {1, second}. + +do_parse_date_str(<<>>, _, Result) -> Result; +do_parse_date_str(_, [], Result) -> Result; +do_parse_date_str(Date, [Key | Formatter], Result) -> + Size = date_size(Key), + <> = Date, + case lists:member(Key, ?DATE_PART) of + true -> + do_parse_date_str(Tail, Formatter, Result#{Key => erlang:binary_to_integer(DatePart)}); + false -> + case lists:member(Key, ?DATE_ZONE_NAME) of + true -> + do_parse_date_str(Tail, Formatter, Result#{parsed_offset => offset_second(DatePart)}); + false -> + do_parse_date_str(Tail, Formatter, Result) + end + end. + +date_size(Str) when is_list(Str) -> erlang:length(Str); +date_size(year) -> 4; +date_size(month) -> 2; +date_size(day) -> 2; +date_size(hour) -> 2; +date_size(minute) -> 2; +date_size(second) -> 2; +date_size(millisecond) -> 3; +date_size(microsecond) -> 6; +date_size(nanosecond) -> 9; +date_size(timezone) -> 5; +date_size(timezone1) -> 6; +date_size(timezone2) -> 9. + +dm(1) -> 0; +dm(2) -> 31; +dm(3) -> 59; +dm(4) -> 90; +dm(5) -> 120; +dm(6) -> 151; +dm(7) -> 181; +dm(8) -> 212; +dm(9) -> 243; +dm(10) -> 273; +dm(11) -> 304; +dm(12) -> 334. + diff --git a/src/emqx_channel.erl b/src/emqx_channel.erl index 530b04250..63a67592f 100644 --- a/src/emqx_channel.erl +++ b/src/emqx_channel.erl @@ -1285,8 +1285,6 @@ auth_connect(#mqtt_packet_connect{password = Password}, username := Username} = ClientInfo, case emqx_access_control:authenticate(ClientInfo#{password => Password}) of {ok, AuthResult} -> - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), NClientInfo = maps:merge(ClientInfo, AuthResult), {ok, Channel#channel{clientinfo = NClientInfo}}; {error, Reason} -> @@ -1295,9 +1293,6 @@ auth_connect(#mqtt_packet_connect{password = Password}, {error, emqx_reason_codes:connack_error(Reason)} end. -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. - %%-------------------------------------------------------------------- %% Enhanced Authentication diff --git a/src/emqx_logger_textfmt.erl b/src/emqx_logger_textfmt.erl index 98104f58c..1d1bb8dcd 100644 --- a/src/emqx_logger_textfmt.erl +++ b/src/emqx_logger_textfmt.erl @@ -28,11 +28,27 @@ , gl % not interesting ]). -check_config(X) -> logger_formatter:check_config(X). +check_config(Config0) -> + Config = maps:without([date_format], Config0), + logger_formatter:check_config(Config). -format(#{msg := Msg0, meta := Meta} = Event, Config) -> +format(#{msg := Msg0, meta := Meta} = Event, + #{date_format := rfc3339, template := Template0} = Config) -> Msg = maybe_merge(Msg0, Meta), - logger_formatter:format(Event#{msg := Msg}, Config). + Template = [time | Template0], + logger_formatter:format(Event#{msg := Msg}, Config#{template => Template}); +format(#{msg := Msg0, meta := Meta} = Event, + #{date_format := DFS} = Config) -> + Msg = maybe_merge(Msg0, Meta), + Time = + case maps:get(time, Event, undefined) of + undefined -> + erlang:system_time(microsecond); + T -> + T + end, + Date = emqx_calendar:format(Time, microsecond, local, DFS), + [Date | logger_formatter:format(Event#{msg := Msg}, Config)]. maybe_merge({report, Report}, Meta) when is_map(Report) -> {report, maps:merge(Report, filter(Meta))}; diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl index 9bb19b30a..7bec83465 100644 --- a/src/emqx_metrics.erl +++ b/src/emqx_metrics.erl @@ -68,8 +68,10 @@ %% BACKW -export([%% v4.3.0 upgrade_retained_delayed_counter_type/0, - %% e4.4.0, e4.3.0-e4.3.6, v4.3.0-v4.3.11 - assign_acl_stats_from_ets_to_counter/0 + %% v4.3.0-v4.3.11, e4.3.0-e4.3.6; v4.4.0, e4.4.0 + assign_acl_stats_from_ets_to_counter/0, + %% v4.3.0-v4.3.14, e4.3.0-e4.3.9; v4.4.0-v4.4.3, e4.4.0-e4.4.3, + assign_auth_stats_from_ets_to_counter/0 ]). -export_type([metric_idx/0]). @@ -174,7 +176,6 @@ {counter, 'client.connack'}, {counter, 'client.connected'}, {counter, 'client.authenticate'}, - {counter, 'client.auth.anonymous'}, {counter, 'client.check_acl'}, {counter, 'client.subscribe'}, {counter, 'client.unsubscribe'}, @@ -189,8 +190,16 @@ {counter, 'session.discarded'}, {counter, 'session.terminated'} ]). -%% Statistic metrics for ACL checking --define(STASTS_ACL_METRICS, + +%% Statistic metrics for auth checking +-define(STATS_AUTH_METRICS, + [ {counter, 'client.auth.success'}, + {counter, 'client.auth.success.anonymous'}, + {counter, 'client.auth.failure'} + ]). + +%% Statistic metrics for ACL checking stats +-define(STATS_ACL_METRICS, [ {counter, 'client.acl.allow'}, {counter, 'client.acl.deny'}, {counter, 'client.acl.cache_hit'} @@ -228,6 +237,21 @@ assign_acl_stats_from_ets_to_counter() -> ok = counters:put(CRef, Idx, Val) end, Names). +%% BACKW: %% v4.3.0-v4.3.14, e4.3.0-e4.3.9; v4.4.0-v4.4.3, e4.4.0-e4.4.3, +assign_auth_stats_from_ets_to_counter() -> + CRef = persistent_term:get(?MODULE), + Names = ['client.auth.success', 'client.auth.success.anonymous', 'client.auth.failure'], + lists:foreach(fun(Name) -> + Val = case emqx_metrics:val(Name) of + undefined -> 0; + Val0 -> Val0 + end, + Idx = reserved_idx(Name), + Metric = #metric{name = Name, type = counter, idx = Idx}, + ok = gen_server:call(?SERVER, {set, Metric}), + ok = counters:put(CRef, Idx, Val) + end, Names). + %%-------------------------------------------------------------------- %% Metrics API %%-------------------------------------------------------------------- @@ -458,7 +482,8 @@ init([]) -> ?DELIVERY_METRICS, ?CLIENT_METRICS, ?SESSION_METRICS, - ?STASTS_ACL_METRICS + ?STATS_AUTH_METRICS, + ?STATS_ACL_METRICS ]), % Store reserved indices ok = lists:foreach(fun({Type, Name}) -> @@ -592,7 +617,6 @@ reserved_idx('client.connack') -> 201; reserved_idx('client.connected') -> 202; reserved_idx('client.authenticate') -> 203; reserved_idx('client.enhanced_authenticate') -> 204; -reserved_idx('client.auth.anonymous') -> 205; reserved_idx('client.check_acl') -> 206; reserved_idx('client.subscribe') -> 207; reserved_idx('client.unsubscribe') -> 208; @@ -604,9 +628,13 @@ reserved_idx('session.takeovered') -> 222; reserved_idx('session.discarded') -> 223; reserved_idx('session.terminated') -> 224; %% Stats metrics +%% ACL reserved_idx('client.acl.allow') -> 300; reserved_idx('client.acl.deny') -> 301; reserved_idx('client.acl.cache_hit') -> 302; +%% Auth +reserved_idx('client.auth.success') -> 310; +reserved_idx('client.auth.success.anonymous') -> 311; +reserved_idx('client.auth.failure') -> 312; reserved_idx(_) -> undefined. - diff --git a/src/emqx_os_mon.erl b/src/emqx_os_mon.erl index b23f3e65d..bbf4fd80e 100644 --- a/src/emqx_os_mon.erl +++ b/src/emqx_os_mon.erl @@ -78,32 +78,29 @@ set_cpu_low_watermark(Float) -> call({set_cpu_low_watermark, Float}). get_mem_check_interval() -> - memsup:get_check_interval() div 1000. + call(?FUNCTION_NAME). -set_mem_check_interval(Seconds) when Seconds < 60 -> - memsup:set_check_interval(1); set_mem_check_interval(Seconds) -> - memsup:set_check_interval(Seconds div 60). + call({?FUNCTION_NAME, Seconds}). get_sysmem_high_watermark() -> - memsup:get_sysmem_high_watermark(). + call(?FUNCTION_NAME). -set_sysmem_high_watermark(Float) -> - V = Float/100, +set_sysmem_high_watermark(HW) -> case load_ctl:get_config() of #{ ?MEM_MON_F0 := true } = OldLC -> ok = load_ctl:put_config(OldLC#{ ?MEM_MON_F0 => true - , ?MEM_MON_F1 => V}); + , ?MEM_MON_F1 => HW / 100}); _ -> skip end, - memsup:set_sysmem_high_watermark(V). + gen_server:call(?OS_MON, {?FUNCTION_NAME, HW}, infinity). get_procmem_high_watermark() -> memsup:get_procmem_high_watermark(). -set_procmem_high_watermark(Float) -> - memsup:set_procmem_high_watermark(Float / 100). +set_procmem_high_watermark(HW) -> + memsup:set_procmem_high_watermark(HW / 100). call(Req) -> gen_server:call(?OS_MON, Req, infinity). @@ -113,16 +110,38 @@ call(Req) -> %%-------------------------------------------------------------------- init([Opts]) -> - set_mem_check_interval(proplists:get_value(mem_check_interval, Opts)), - SysHW = proplists:get_value(sysmem_high_watermark, Opts), - set_sysmem_high_watermark(SysHW), + process_flag(trap_exit, true), + %% make sure memsup will not emit system memory alarms + memsup:set_sysmem_high_watermark(1), set_procmem_high_watermark(proplists:get_value(procmem_high_watermark, Opts)), - ensure_system_memory_alarm(SysHW), - {ok, ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts), + MemCheckInterval = do_resolve_mem_check_interval(proplists:get_value(mem_check_interval, Opts)), + SysHW = proplists:get_value(sysmem_high_watermark, Opts), + St = ensure_check_timer(#{cpu_high_watermark => proplists:get_value(cpu_high_watermark, Opts), cpu_low_watermark => proplists:get_value(cpu_low_watermark, Opts), cpu_check_interval => proplists:get_value(cpu_check_interval, Opts), - timer => undefined})}. + sysmem_high_watermark => SysHW, + mem_check_interval => MemCheckInterval, + timer => undefined}), + ok = do_set_mem_check_interval(MemCheckInterval), + %% update immediately after start/restart + ok = update_mem_alarm_status(SysHW), + {ok, ensure_mem_check_timer(St)}. +handle_call(get_sysmem_high_watermark, _From, State) -> + #{sysmem_high_watermark := SysHW} = State, + {reply, maybe_round(SysHW), State}; +handle_call(get_mem_check_interval, _From, State) -> + #{mem_check_interval := Interval} = State, + {reply, Interval, State}; +handle_call({set_sysmem_high_watermark, SysHW}, _From, State) -> + %% update immediately after start/restart + ok = update_mem_alarm_status(SysHW), + {reply, ok, State#{sysmem_high_watermark => SysHW}}; +handle_call({set_mem_check_interval, Seconds0}, _From, State) -> + Seconds = do_resolve_mem_check_interval(Seconds0), + ok = do_set_mem_check_interval(Seconds), + %% will start taking effect when the current timer expires + {reply, ok, State#{mem_check_interval => Seconds}}; handle_call(get_cpu_check_interval, _From, State) -> {reply, maps:get(cpu_check_interval, State, undefined), State}; @@ -168,16 +187,28 @@ handle_info({timeout, Timer, check}, State = #{timer := Timer, ensure_check_timer(State) end, {noreply, NState}; - +handle_info({timeout, Timer, check_mem}, #{mem_check_timer := Timer, + sysmem_high_watermark := SysHW + } = State) -> + ok = update_mem_alarm_status(SysHW), + NState = ensure_mem_check_timer(State#{mem_check_timer := undefined}), + {noreply, NState}; handle_info(Info, State) -> ?LOG(error, "unexpected info: ~p", [Info]), {noreply, State}. -terminate(_Reason, #{timer := Timer}) -> +terminate(_Reason, #{timer := Timer} = St) -> + emqx_misc:cancel_timer(maps:get(mem_check_timer, St, undefined)), emqx_misc:cancel_timer(Timer). code_change(_OldVsn, State, _Extra) -> - {ok, State}. + %% NOTE: downgrade is not handled as the extra fields added to State + %% does not affect old version code. + %% The only thing which may slip through is that a started timer + %% will result in a "unexpected info" error log for the old version code + NewState = ensure_mem_check_timer(State), + SysHW = resolve_sysmem_high_watermark(State), + {ok, NewState#{sysmem_high_watermark => SysHW}}. %%-------------------------------------------------------------------- %% Internal functions @@ -189,19 +220,99 @@ ensure_check_timer(State = #{cpu_check_interval := Interval}) -> _ -> State#{timer := emqx_misc:start_timer(timer:seconds(Interval), check)} end. -%% At startup, memsup starts first and checks for memory alarms, -%% but emqx_alarm_handler is not yet used instead of alarm_handler, -%% so alarm_handler is used directly for notification (normally emqx_alarm_handler should be used). -%%The internal memsup will no longer trigger events that have been alerted, -%% and there is no exported function to remove the alerted flag, -%% so it can only be checked again at startup. -ensure_system_memory_alarm(HW) -> - case erlang:whereis(memsup) of - undefined -> ok; - _Pid -> - {Total, Allocated, _Worst} = memsup:get_memory_data(), - case Total =/= 0 andalso Allocated/Total * 100 > HW of - true -> emqx_alarm:activate(high_system_memory_usage, #{high_watermark => HW}); - false -> ok - end +ensure_mem_check_timer(#{mem_check_timer := Ref} = State) when is_reference(Ref) -> + %% timer already started + State; +ensure_mem_check_timer(State) -> + Interval = resolve_mem_check_interval(State), + case is_sysmem_check_supported() of + true -> + State#{mem_check_timer => emqx_misc:start_timer(timer:seconds(Interval), check_mem), + mem_check_interval => Interval + }; + false -> + State#{mem_check_timer => undefined, + mem_check_interval => Interval + } + end. + +resolve_mem_check_interval(#{mem_check_interval := Seconds}) when is_integer(Seconds) -> + Seconds; +resolve_mem_check_interval(_) -> + %% this only happens when hot-upgrade from older version (< 4.3.14, or < 4.4.4) + try + %% memsup has interval set API using minutes, but returns in milliseconds from get API + IntervalMs = memsup:get_check_interval(), + true = (IntervalMs > 1000), + IntervalMs div 1000 + catch + _ : _ -> + %% this is the memsup default + 60 + end. + +is_sysmem_check_supported() -> + %% sorry Mac and Windows, for now + {unix, linux} =:= os:type(). + +%% we still need to set memsup interval for process (not system) memory check +do_set_mem_check_interval(Seconds) -> + Minutes = Seconds div 60, + _ = memsup:set_check_interval(Minutes), + ok. + +%% keep the time unit alignment with memsup, minmum interval is 60 seconds. +do_resolve_mem_check_interval(Seconds) -> + case is_integer(Seconds) andalso Seconds >= 60 of + true -> Seconds; + false -> 60 + end. + +resolve_sysmem_high_watermark(#{sysmem_high_watermark := SysHW}) -> SysHW; +resolve_sysmem_high_watermark(_) -> + %% sysmem_high_watermark is not found in state map + %% get it from memsup + memsup:get_sysmem_high_watermark(). + +update_mem_alarm_status(SysHW) -> + case is_sysmem_check_supported() of + true -> + do_update_mem_alarm_status(SysHW); + false -> + %% in case the old alarm is activated + ok = emqx_alarm:ensure_deactivated(high_system_memory_usage, #{reason => disabled}) + end. + +do_update_mem_alarm_status(SysHW) -> + Usage = current_sysmem_percent(), + case Usage > SysHW of + true -> + _ = emqx_alarm:activate( + high_system_memory_usage, + #{ + usage => Usage, + high_watermark => SysHW + } + ); + _ -> + ok = emqx_alarm:ensure_deactivated( + high_system_memory_usage, + #{ + usage => Usage, + high_watermark => SysHW + } + ) + end, + ok. + +current_sysmem_percent() -> + Ratio = load_ctl:get_memory_usage(), + erlang:floor(Ratio * 10000) / 100. + +maybe_round(X) when is_integer(X) -> X; +maybe_round(X) when is_float(X) -> + R = erlang:round(X), + case erlang:abs(X - R) > 1.0e-6 of + true -> X; + false -> R end. diff --git a/test/emqx_os_mon_SUITE.erl b/test/emqx_os_mon_SUITE.erl index 575c97225..d4b8a5d2a 100644 --- a/test/emqx_os_mon_SUITE.erl +++ b/test/emqx_os_mon_SUITE.erl @@ -43,14 +43,23 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([]), application:stop(os_mon). -% t_set_mem_check_interval(_) -> -% error('TODO'). +t_set_mem_check_interval(_) -> + emqx_os_mon:set_mem_check_interval(0), + ?assertEqual(60, emqx_os_mon:get_mem_check_interval()), + emqx_os_mon:set_mem_check_interval(61), + ?assertEqual(61, emqx_os_mon:get_mem_check_interval()), + ok. -% t_set_sysmem_high_watermark(_) -> -% error('TODO'). - -% t_set_procmem_high_watermark(_) -> -% error('TODO'). +t_set_sysmem_high_watermark(_) -> + emqx_os_mon:set_sysmem_high_watermark(10), + ?assertEqual(10, emqx_os_mon:get_sysmem_high_watermark()), + emqx_os_mon:set_sysmem_high_watermark(100), + ?assertEqual(100, emqx_os_mon:get_sysmem_high_watermark()), + emqx_os_mon:set_sysmem_high_watermark(90), + ?assertEqual(90, emqx_os_mon:get_sysmem_high_watermark()), + emqx_os_mon:set_sysmem_high_watermark(93.2), + ?assertEqual(93.2, emqx_os_mon:get_sysmem_high_watermark()), + ok. t_api(_) -> ?assertEqual(1, emqx_os_mon:get_cpu_check_interval()),