diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..c563aa10d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,27 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +charset = utf-8 +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + + +# Matches multiple files with brace expansion notation +# Set default charset +[*.{erl, src, hrl}] +indent_style = space +indent_size = 4 + +# Tab indentation (no size specified) +[Makefile] +indent_style = tab + +# Matches the exact files either package.json or .travis.yml +[{.travis.yml}] +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore index f8f1af339..7a4e891d1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,17 +17,24 @@ log/ *.so .erlang.mk/ cover/ -emqttd.d +emqx.d eunit.coverdata test/ct.cover.spec logs ct.coverdata .idea/ -emqttd.iml +emqx.iml _rel/ data/ _build .rebar3 rebar3.crashdump .DS_Store -rebar.config +emqx.iml +bbmustache/ +etc/gen.emqx.conf +compile_commands.json +cuttlefish +rebar.lock +xrefr +erlang.mk diff --git a/.travis.yml b/.travis.yml index 8d4f7acc4..8960ab7c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,20 @@ language: erlang otp_release: - - 20.0 + - 21.0.4 + +before_install: + - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd .. script: - - make + - make dep-vsn-check + - make rebar-compile + - make rebar-xref + - make rebar-eunit + - make rebar-ct + - make rebar-cover + +after_success: + - make coveralls sudo: false diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md deleted file mode 100644 index 0e2b17bb8..000000000 --- a/CONTRIBUTORS.md +++ /dev/null @@ -1,26 +0,0 @@ - -* [@callbay](https://github.com/callbay) - -* [@lsxredrain](https://github.com/lsxredrain) - -* [@hejin1026](https://github.com/hejin1026) - -* [@desoulter](https://github.com/desoulter) - -* [@turtleDeng](https://github.com/turtleDeng) - -* [@Hades32](https://github.com/Hades32) - -* [@huangdan](https://github.com/huangdan) - -* [@phanimahesh](https://github.com/phanimahesh) - -* [@dvliman](https://github.com/dvliman) - -* [@vowstar](https://github.com/vowstar) - -* [@TheWaWaR](https://github.com/TheWaWaR) - -* [@hejin1026](https://github.com/hejin1026) - -* [@farhadi](https://github.com/farhadi) diff --git a/LICENSE-MPL-RabbitMQ b/LICENSE-MPL-RabbitMQ deleted file mode 100644 index f1ba9a5ca..000000000 --- a/LICENSE-MPL-RabbitMQ +++ /dev/null @@ -1,455 +0,0 @@ - MOZILLA PUBLIC LICENSE - Version 1.1 - - --------------- - -1. Definitions. - - 1.0.1. "Commercial Use" means distribution or otherwise making the - Covered Code available to a third party. - - 1.1. "Contributor" means each entity that creates or contributes to - the creation of Modifications. - - 1.2. "Contributor Version" means the combination of the Original - Code, prior Modifications used by a Contributor, and the Modifications - made by that particular Contributor. - - 1.3. "Covered Code" means the Original Code or Modifications or the - combination of the Original Code and Modifications, in each case - including portions thereof. - - 1.4. "Electronic Distribution Mechanism" means a mechanism generally - accepted in the software development community for the electronic - transfer of data. - - 1.5. "Executable" means Covered Code in any form other than Source - Code. - - 1.6. "Initial Developer" means the individual or entity identified - as the Initial Developer in the Source Code notice required by Exhibit - A. - - 1.7. "Larger Work" means a work which combines Covered Code or - portions thereof with code not governed by the terms of this License. - - 1.8. "License" means this document. - - 1.8.1. "Licensable" means having the right to grant, to the maximum - extent possible, whether at the time of the initial grant or - subsequently acquired, any and all of the rights conveyed herein. - - 1.9. "Modifications" means any addition to or deletion from the - substance or structure of either the Original Code or any previous - Modifications. When Covered Code is released as a series of files, a - Modification is: - A. Any addition to or deletion from the contents of a file - containing Original Code or previous Modifications. - - B. Any new file that contains any part of the Original Code or - previous Modifications. - - 1.10. "Original Code" means Source Code of computer software code - which is described in the Source Code notice required by Exhibit A as - Original Code, and which, at the time of its release under this - License is not already Covered Code governed by this License. - - 1.10.1. "Patent Claims" means any patent claim(s), now owned or - hereafter acquired, including without limitation, method, process, - and apparatus claims, in any patent Licensable by grantor. - - 1.11. "Source Code" means the preferred form of the Covered Code for - making modifications to it, including all modules it contains, plus - any associated interface definition files, scripts used to control - compilation and installation of an Executable, or source code - differential comparisons against either the Original Code or another - well known, available Covered Code of the Contributor's choice. The - Source Code can be in a compressed or archival form, provided the - appropriate decompression or de-archiving software is widely available - for no charge. - - 1.12. "You" (or "Your") means an individual or a legal entity - exercising rights under, and complying with all of the terms of, this - License or a future version of this License issued under Section 6.1. - For legal entities, "You" includes any entity which controls, is - controlled by, or is under common control with You. For purposes of - this definition, "control" means (a) the power, direct or indirect, - to cause the direction or management of such entity, whether by - contract or otherwise, or (b) ownership of more than fifty percent - (50%) of the outstanding shares or beneficial ownership of such - entity. - -2. Source Code License. - - 2.1. The Initial Developer Grant. - The Initial Developer hereby grants You a world-wide, royalty-free, - non-exclusive license, subject to third party intellectual property - claims: - (a) under intellectual property rights (other than patent or - trademark) Licensable by Initial Developer to use, reproduce, - modify, display, perform, sublicense and distribute the Original - Code (or portions thereof) with or without Modifications, and/or - as part of a Larger Work; and - - (b) under Patents Claims infringed by the making, using or - selling of Original Code, to make, have made, use, practice, - sell, and offer for sale, and/or otherwise dispose of the - Original Code (or portions thereof). - - (c) the licenses granted in this Section 2.1(a) and (b) are - effective on the date Initial Developer first distributes - Original Code under the terms of this License. - - (d) Notwithstanding Section 2.1(b) above, no patent license is - granted: 1) for code that You delete from the Original Code; 2) - separate from the Original Code; or 3) for infringements caused - by: i) the modification of the Original Code or ii) the - combination of the Original Code with other software or devices. - - 2.2. Contributor Grant. - Subject to third party intellectual property claims, each Contributor - hereby grants You a world-wide, royalty-free, non-exclusive license - - (a) under intellectual property rights (other than patent or - trademark) Licensable by Contributor, to use, reproduce, modify, - display, perform, sublicense and distribute the Modifications - created by such Contributor (or portions thereof) either on an - unmodified basis, with other Modifications, as Covered Code - and/or as part of a Larger Work; and - - (b) under Patent Claims infringed by the making, using, or - selling of Modifications made by that Contributor either alone - and/or in combination with its Contributor Version (or portions - of such combination), to make, use, sell, offer for sale, have - made, and/or otherwise dispose of: 1) Modifications made by that - Contributor (or portions thereof); and 2) the combination of - Modifications made by that Contributor with its Contributor - Version (or portions of such combination). - - (c) the licenses granted in Sections 2.2(a) and 2.2(b) are - effective on the date Contributor first makes Commercial Use of - the Covered Code. - - (d) Notwithstanding Section 2.2(b) above, no patent license is - granted: 1) for any code that Contributor has deleted from the - Contributor Version; 2) separate from the Contributor Version; - 3) for infringements caused by: i) third party modifications of - Contributor Version or ii) the combination of Modifications made - by that Contributor with other software (except as part of the - Contributor Version) or other devices; or 4) under Patent Claims - infringed by Covered Code in the absence of Modifications made by - that Contributor. - -3. Distribution Obligations. - - 3.1. Application of License. - The Modifications which You create or to which You contribute are - governed by the terms of this License, including without limitation - Section 2.2. The Source Code version of Covered Code may be - distributed only under the terms of this License or a future version - of this License released under Section 6.1, and You must include a - copy of this License with every copy of the Source Code You - distribute. You may not offer or impose any terms on any Source Code - version that alters or restricts the applicable version of this - License or the recipients' rights hereunder. However, You may include - an additional document offering the additional rights described in - Section 3.5. - - 3.2. Availability of Source Code. - Any Modification which You create or to which You contribute must be - made available in Source Code form under the terms of this License - either on the same media as an Executable version or via an accepted - Electronic Distribution Mechanism to anyone to whom you made an - Executable version available; and if made available via Electronic - Distribution Mechanism, must remain available for at least twelve (12) - months after the date it initially became available, or at least six - (6) months after a subsequent version of that particular Modification - has been made available to such recipients. You are responsible for - ensuring that the Source Code version remains available even if the - Electronic Distribution Mechanism is maintained by a third party. - - 3.3. Description of Modifications. - You must cause all Covered Code to which You contribute to contain a - file documenting the changes You made to create that Covered Code and - the date of any change. You must include a prominent statement that - the Modification is derived, directly or indirectly, from Original - Code provided by the Initial Developer and including the name of the - Initial Developer in (a) the Source Code, and (b) in any notice in an - Executable version or related documentation in which You describe the - origin or ownership of the Covered Code. - - 3.4. Intellectual Property Matters - (a) Third Party Claims. - If Contributor has knowledge that a license under a third party's - intellectual property rights is required to exercise the rights - granted by such Contributor under Sections 2.1 or 2.2, - Contributor must include a text file with the Source Code - distribution titled "LEGAL" which describes the claim and the - party making the claim in sufficient detail that a recipient will - know whom to contact. If Contributor obtains such knowledge after - the Modification is made available as described in Section 3.2, - Contributor shall promptly modify the LEGAL file in all copies - Contributor makes available thereafter and shall take other steps - (such as notifying appropriate mailing lists or newsgroups) - reasonably calculated to inform those who received the Covered - Code that new knowledge has been obtained. - - (b) Contributor APIs. - If Contributor's Modifications include an application programming - interface and Contributor has knowledge of patent licenses which - are reasonably necessary to implement that API, Contributor must - also include this information in the LEGAL file. - - (c) Representations. - Contributor represents that, except as disclosed pursuant to - Section 3.4(a) above, Contributor believes that Contributor's - Modifications are Contributor's original creation(s) and/or - Contributor has sufficient rights to grant the rights conveyed by - this License. - - 3.5. Required Notices. - You must duplicate the notice in Exhibit A in each file of the Source - Code. If it is not possible to put such notice in a particular Source - Code file due to its structure, then You must include such notice in a - location (such as a relevant directory) where a user would be likely - to look for such a notice. If You created one or more Modification(s) - You may add your name as a Contributor to the notice described in - Exhibit A. You must also duplicate this License in any documentation - for the Source Code where You describe recipients' rights or ownership - rights relating to Covered Code. You may choose to offer, and to - charge a fee for, warranty, support, indemnity or liability - obligations to one or more recipients of Covered Code. However, You - may do so only on Your own behalf, and not on behalf of the Initial - Developer or any Contributor. You must make it absolutely clear than - any such warranty, support, indemnity or liability obligation is - offered by You alone, and You hereby agree to indemnify the Initial - Developer and every Contributor for any liability incurred by the - Initial Developer or such Contributor as a result of warranty, - support, indemnity or liability terms You offer. - - 3.6. Distribution of Executable Versions. - You may distribute Covered Code in Executable form only if the - requirements of Section 3.1-3.5 have been met for that Covered Code, - and if You include a notice stating that the Source Code version of - the Covered Code is available under the terms of this License, - including a description of how and where You have fulfilled the - obligations of Section 3.2. The notice must be conspicuously included - in any notice in an Executable version, related documentation or - collateral in which You describe recipients' rights relating to the - Covered Code. You may distribute the Executable version of Covered - Code or ownership rights under a license of Your choice, which may - contain terms different from this License, provided that You are in - compliance with the terms of this License and that the license for the - Executable version does not attempt to limit or alter the recipient's - rights in the Source Code version from the rights set forth in this - License. If You distribute the Executable version under a different - license You must make it absolutely clear that any terms which differ - from this License are offered by You alone, not by the Initial - Developer or any Contributor. You hereby agree to indemnify the - Initial Developer and every Contributor for any liability incurred by - the Initial Developer or such Contributor as a result of any such - terms You offer. - - 3.7. Larger Works. - You may create a Larger Work by combining Covered Code with other code - not governed by the terms of this License and distribute the Larger - Work as a single product. In such a case, You must make sure the - requirements of this License are fulfilled for the Covered Code. - -4. Inability to Comply Due to Statute or Regulation. - - If it is impossible for You to comply with any of the terms of this - License with respect to some or all of the Covered Code due to - statute, judicial order, or regulation then You must: (a) comply with - the terms of this License to the maximum extent possible; and (b) - describe the limitations and the code they affect. Such description - must be included in the LEGAL file described in Section 3.4 and must - be included with all distributions of the Source Code. Except to the - extent prohibited by statute or regulation, such description must be - sufficiently detailed for a recipient of ordinary skill to be able to - understand it. - -5. Application of this License. - - This License applies to code to which the Initial Developer has - attached the notice in Exhibit A and to related Covered Code. - -6. Versions of the License. - - 6.1. New Versions. - Netscape Communications Corporation ("Netscape") may publish revised - and/or new versions of the License from time to time. Each version - will be given a distinguishing version number. - - 6.2. Effect of New Versions. - Once Covered Code has been published under a particular version of the - License, You may always continue to use it under the terms of that - version. You may also choose to use such Covered Code under the terms - of any subsequent version of the License published by Netscape. No one - other than Netscape has the right to modify the terms applicable to - Covered Code created under this License. - - 6.3. Derivative Works. - If You create or use a modified version of this License (which you may - only do in order to apply it to code which is not already Covered Code - governed by this License), You must (a) rename Your license so that - the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", - "MPL", "NPL" or any confusingly similar phrase do not appear in your - license (except to note that your license differs from this License) - and (b) otherwise make it clear that Your version of the license - contains terms which differ from the Mozilla Public License and - Netscape Public License. (Filling in the name of the Initial - Developer, Original Code or Contributor in the notice described in - Exhibit A shall not of themselves be deemed to be modifications of - this License.) - -7. DISCLAIMER OF WARRANTY. - - COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, - WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, - WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF - DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. - THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE - IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, - YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE - COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER - OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF - ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -8. TERMINATION. - - 8.1. This License and the rights granted hereunder will terminate - automatically if You fail to comply with terms herein and fail to cure - such breach within 30 days of becoming aware of the breach. All - sublicenses to the Covered Code which are properly granted shall - survive any termination of this License. Provisions which, by their - nature, must remain in effect beyond the termination of this License - shall survive. - - 8.2. If You initiate litigation by asserting a patent infringement - claim (excluding declatory judgment actions) against Initial Developer - or a Contributor (the Initial Developer or Contributor against whom - You file such action is referred to as "Participant") alleging that: - - (a) such Participant's Contributor Version directly or indirectly - infringes any patent, then any and all rights granted by such - Participant to You under Sections 2.1 and/or 2.2 of this License - shall, upon 60 days notice from Participant terminate prospectively, - unless if within 60 days after receipt of notice You either: (i) - agree in writing to pay Participant a mutually agreeable reasonable - royalty for Your past and future use of Modifications made by such - Participant, or (ii) withdraw Your litigation claim with respect to - the Contributor Version against such Participant. If within 60 days - of notice, a reasonable royalty and payment arrangement are not - mutually agreed upon in writing by the parties or the litigation claim - is not withdrawn, the rights granted by Participant to You under - Sections 2.1 and/or 2.2 automatically terminate at the expiration of - the 60 day notice period specified above. - - (b) any software, hardware, or device, other than such Participant's - Contributor Version, directly or indirectly infringes any patent, then - any rights granted to You by such Participant under Sections 2.1(b) - and 2.2(b) are revoked effective as of the date You first made, used, - sold, distributed, or had made, Modifications made by that - Participant. - - 8.3. If You assert a patent infringement claim against Participant - alleging that such Participant's Contributor Version directly or - indirectly infringes any patent where such claim is resolved (such as - by license or settlement) prior to the initiation of patent - infringement litigation, then the reasonable value of the licenses - granted by such Participant under Sections 2.1 or 2.2 shall be taken - into account in determining the amount or value of any payment or - license. - - 8.4. In the event of termination under Sections 8.1 or 8.2 above, - all end user license agreements (excluding distributors and resellers) - which have been validly granted by You or any distributor hereunder - prior to termination shall survive termination. - -9. LIMITATION OF LIABILITY. - - UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT - (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL - DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, - OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR - ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY - CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, - WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER - COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN - INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF - LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY - RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW - PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE - EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO - THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. - -10. U.S. GOVERNMENT END USERS. - - The Covered Code is a "commercial item," as that term is defined in - 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer - software" and "commercial computer software documentation," as such - terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 - C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), - all U.S. Government End Users acquire Covered Code with only those - rights set forth herein. - -11. MISCELLANEOUS. - - This License represents the complete agreement concerning subject - matter hereof. If any provision of this License is held to be - unenforceable, such provision shall be reformed only to the extent - necessary to make it enforceable. This License shall be governed by - California law provisions (except to the extent applicable law, if - any, provides otherwise), excluding its conflict-of-law provisions. - With respect to disputes in which at least one party is a citizen of, - or an entity chartered or registered to do business in the United - States of America, any litigation relating to this License shall be - subject to the jurisdiction of the Federal Courts of the Northern - District of California, with venue lying in Santa Clara County, - California, with the losing party responsible for costs, including - without limitation, court costs and reasonable attorneys' fees and - expenses. The application of the United Nations Convention on - Contracts for the International Sale of Goods is expressly excluded. - Any law or regulation which provides that the language of a contract - shall be construed against the drafter shall not apply to this - License. - -12. RESPONSIBILITY FOR CLAIMS. - - As between Initial Developer and the Contributors, each party is - responsible for claims and damages arising, directly or indirectly, - out of its utilization of rights under this License and You agree to - work with Initial Developer and Contributors to distribute such - responsibility on an equitable basis. Nothing herein is intended or - shall be deemed to constitute any admission of liability. - -13. MULTIPLE-LICENSED CODE. - - Initial Developer may designate portions of the Covered Code as - "Multiple-Licensed". "Multiple-Licensed" means that the Initial - Developer permits you to utilize portions of the Covered Code under - Your choice of the NPL or the alternative licenses, if any, specified - by the Initial Developer in the file described in Exhibit A. - -EXHIBIT A -Mozilla Public License. - - ``The contents of this file are subject to the Mozilla Public License - Version 1.1 (the "License"); you may not use this file except in - compliance with the License. You may obtain a copy of the License at - http://www.mozilla.org/MPL/ - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the - License for the specific language governing rights and limitations - under the License. - - The Original Code is RabbitMQ. - - The Initial Developer of the Original Code is Pivotal Software, Inc. - Copyright (c) 2007-2016 Pivotal Software, Inc. All rights reserved.'' - - [NOTE: The text of this Exhibit A may differ slightly from the text of - the notices in the Source Code files of the Original Code. You should - use the text of this Exhibit A rather than the text found in the - Original Code Source Code for Your Modifications.] diff --git a/Makefile b/Makefile index 3699c4efc..69312a1c2 100644 --- a/Makefile +++ b/Makefile @@ -1,57 +1,136 @@ -PROJECT = emqttd -PROJECT_DESCRIPTION = Erlang MQTT Broker -PROJECT_VERSION = 2.3.11 +.PHONY: plugins tests -DEPS = goldrush gproc lager esockd ekka mochiweb pbkdf2 lager_syslog bcrypt clique jsx +PROJECT = emqx +PROJECT_DESCRIPTION = EMQ X Broker -dep_goldrush = git https://github.com/basho/goldrush 0.1.9 -dep_gproc = git https://github.com/uwiger/gproc 0.8.0 -dep_getopt = git https://github.com/jcomellas/getopt v0.8.2 -dep_lager = git https://github.com/basho/lager 3.2.4 -dep_esockd = git https://github.com/emqtt/esockd v5.2.2 -dep_ekka = git https://github.com/emqtt/ekka v0.2.3 -dep_mochiweb = git https://github.com/emqtt/mochiweb v4.2.2 -dep_pbkdf2 = git https://github.com/emqtt/pbkdf2 2.0.1 -dep_lager_syslog = git https://github.com/basho/lager_syslog 3.0.1 -dep_bcrypt = git https://github.com/smarkets/erlang-bcrypt master -dep_clique = git https://github.com/emqtt/clique v0.3.10 -dep_jsx = git https://github.com/talentdeficit/jsx v2.8.3 +DEPS = jsx gproc gen_rpc ekka esockd cowboy -ERLC_OPTS += +debug_info -ERLC_OPTS += +'{parse_transform, lager_transform}' +dep_jsx = hex-emqx 2.9.0 +dep_gproc = hex-emqx 0.8.0 +dep_gen_rpc = git-emqx https://github.com/emqx/gen_rpc 2.3.0 +dep_esockd = git-emqx https://github.com/emqx/esockd v5.4.3 +dep_ekka = git-emqx https://github.com/emqx/ekka v0.5.1 +dep_cowboy = hex-emqx 2.4.0 NO_AUTOPATCH = cuttlefish +ERLC_OPTS += +debug_info -DAPPLICATION=emqx + BUILD_DEPS = cuttlefish -dep_cuttlefish = git https://github.com/emqtt/cuttlefish v2.0.11 +dep_cuttlefish = git-emqx https://github.com/emqx/cuttlefish v2.2.0 -TEST_DEPS = emqttc emq_dashboard -dep_emqttc = git https://github.com/emqtt/emqttc -dep_emq_dashboard = git https://github.com/emqtt/emq_dashboard develop +#TEST_DEPS = emqx_ct_helplers +#dep_emqx_ct_helplers = git git@github.com:emqx/emqx-ct-helpers -TEST_ERLC_OPTS += +debug_info -TEST_ERLC_OPTS += +'{parse_transform, lager_transform}' +TEST_ERLC_OPTS += +debug_info -DAPPLICATION=emqx EUNIT_OPTS = verbose -# EUNIT_ERL_OPTS = -CT_SUITES = emqttd emqttd_access emqttd_lib emqttd_inflight emqttd_mod \ - emqttd_net emqttd_mqueue emqttd_protocol emqttd_topic \ - emqttd_router emqttd_trie emqttd_vm emqttd_config +# CT_SUITES = emqx_frame +## emqx_trie emqx_router emqx_frame emqx_mqtt_compat -CT_OPTS = -cover test/ct.cover.spec -erl_args -name emqttd_ct@127.0.0.1 +CT_SUITES = emqx emqx_client emqx_zone emqx_banned emqx_session \ + emqx_access emqx_broker emqx_cm emqx_frame emqx_guid emqx_inflight emqx_json \ + emqx_keepalive emqx_lib emqx_metrics emqx_mod emqx_mod_sup emqx_mqtt_caps \ + emqx_mqtt_props emqx_mqueue emqx_net emqx_pqueue emqx_router emqx_sm \ + emqx_tables emqx_time emqx_topic emqx_trie emqx_vm emqx_mountpoint \ + emqx_listeners emqx_protocol emqx_pool emqx_shared_sub emqx_bridge \ + emqx_hooks emqx_batch emqx_sequence emqx_pmon emqx_pd emqx_gc + +CT_NODE_NAME = emqxct@127.0.0.1 +CT_OPTS = -cover test/ct.cover.spec -erl_args -name $(CT_NODE_NAME) COVER = true -PLT_APPS = sasl asn1 ssl syntax_tools runtime_tools crypto xmerl os_mon inets public_key ssl lager compiler mnesia +PLT_APPS = sasl asn1 ssl syntax_tools runtime_tools crypto xmerl os_mon inets public_key ssl compiler mnesia DIALYZER_DIRS := ebin/ -DIALYZER_OPTS := --verbose --statistics -Werror_handling \ - -Wrace_conditions #-Wunmatched_returns +DIALYZER_OPTS := --verbose --statistics -Werror_handling -Wrace_conditions #-Wunmatched_returns +$(shell [ -f erlang.mk ] || curl -s -o erlang.mk https://raw.githubusercontent.com/emqx/erlmk/master/erlang.mk) include erlang.mk -app:: rebar.config +clean:: gen-clean -app.config:: - ./deps/cuttlefish/cuttlefish -l info -e etc/ -c etc/emq.conf -i priv/emq.schema -d data/ +.PHONY: gen-clean +gen-clean: + @rm -rf bbmustache + @rm -f etc/gen.emqx.conf +bbmustache: + $(verbose) git clone https://github.com/soranoba/bbmustache.git && cd bbmustache && ./rebar3 compile && cd .. + +# This hack is to generate a conf file for testing +# relx overlay is used for release +etc/gen.emqx.conf: bbmustache etc/emqx.conf + $(verbose) erl -noshell -pa bbmustache/_build/default/lib/bbmustache/ebin -eval \ + "{ok, Temp} = file:read_file('etc/emqx.conf'), \ + {ok, Vars0} = file:consult('vars'), \ + Vars = [{atom_to_list(N), list_to_binary(V)} || {N, V} <- Vars0], \ + Targ = bbmustache:render(Temp, Vars), \ + ok = file:write_file('etc/gen.emqx.conf', Targ), \ + halt(0)." + +CUTTLEFISH_SCRIPT = _build/default/lib/cuttlefish/cuttlefish + +app.config: $(CUTTLEFISH_SCRIPT) etc/gen.emqx.conf + $(verbose) $(CUTTLEFISH_SCRIPT) -l info -e etc/ -c etc/gen.emqx.conf -i priv/emqx.schema -d data/ + +ct: app.config + +rebar-cover: + @rebar3 cover + +coveralls: + @rebar3 coveralls send + + +$(CUTTLEFISH_SCRIPT): rebar-deps + @if [ ! -f cuttlefish ]; then make -C _build/default/lib/cuttlefish; fi + +rebar-xref: + @rebar3 xref + +rebar-deps: + @rebar3 get-deps + +rebar-eunit: $(CUTTLEFISH_SCRIPT) + @rebar3 eunit + +rebar-compile: + @rebar3 compile + +rebar-ct: app.config + @rebar3 as test compile + @ln -s -f '../../../../etc' _build/test/lib/emqx/ + @ln -s -f '../../../../data' _build/test/lib/emqx/ + @rebar3 ct -v --readable=false --name $(CT_NODE_NAME) --suite=$(shell echo $(foreach var,$(CT_SUITES),test/$(var)_SUITE) | tr ' ' ',') + +rebar-clean: + @rebar3 clean + +distclean:: + @rm -rf _build cover deps logs log data + @rm -f rebar.lock compile_commands.json cuttlefish + +## Below are for version consistency check during erlang.mk and rebar3 dual mode support +none= +space = $(none) $(none) +comma = , +quote = \" +curly_l = "{" +curly_r = "}" +dep-versions = [$(foreach dep,$(DEPS) $(BUILD_DEPS),$(curly_l)$(dep),$(quote)$(word $(words $(dep_$(dep))),$(dep_$(dep)))$(quote)$(curly_r)$(comma))[]] + +.PHONY: dep-vsn-check +dep-vsn-check: + $(verbose) erl -noshell -eval \ + "MkVsns = lists:sort(lists:flatten($(dep-versions))), \ + {ok, Conf} = file:consult('rebar.config'), \ + {_, Deps1} = lists:keyfind(deps, 1, Conf), \ + {_, Deps2} = lists:keyfind(github_emqx_deps, 1, Conf), \ + F = fun({N, V}) when is_list(V) -> {N, V}; ({N, {git, _, {branch, V}}}) -> {N, V} end, \ + RebarVsns = lists:sort(lists:map(F, Deps1 ++ Deps2)), \ + case {RebarVsns -- MkVsns, MkVsns -- RebarVsns} of \ + {[], []} -> halt(0); \ + {Rebar, Mk} -> erlang:error({deps_version_discrepancy, [{rebar, Rebar}, {mk, Mk}]}) \ + end." diff --git a/README.md b/README.md index ecb5c569f..8d03df8dd 100644 --- a/README.md +++ b/README.md @@ -1,121 +1,85 @@ +# EMQ X Broker -# *EMQ* - Erlang MQTT Broker +*EMQ X* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients. -[![Build Status](https://travis-ci.org/emqtt/emqttd.svg?branch=master)](https://travis-ci.org/emqtt/emqttd) +Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1, as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster. -*EMQ* (Erlang MQTT Broker) is a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OTP. -*EMQ* is fully open source and licensed under the Apache Version 2.0. *EMQ* implements both MQTT V3.1 and V3.1.1 protocol specifications, and supports MQTT-SN, CoAP, WebSocket, STOMP and SockJS at the same time. +- For full list of new features, please read *EMQ X* broker 3.0 [release notes](https://github.com/emqx/emqx/releases/). +- For more information, please visit [EMQ X homepage](http://emqtt.io). -*EMQ* provides a scalable, reliable, enterprise-grade MQTT message Hub for IoT, M2M, Smart Hardware and Mobile Messaging Applications. - -The 1.0 release of the EMQ broker has scaled to 1.3 million concurrent MQTT connections on a 12 Core, 32G CentOS server. - -Please visit [emqtt.io](http://emqtt.io) for more service. Follow us on Twitter: [@emqtt](https://twitter.com/emqtt) - -## Features - -* Full MQTT V3.1/V3.1.1 support -* QoS0, QoS1, QoS2 Publish/Subscribe -* Session Management and Offline Messages -* Retained Message -* Last Will Message -* TCP/SSL Connection -* MQTT Over WebSocket(SSL) -* HTTP Publish API -* MQTT-SN Protocol -* STOMP protocol -* STOMP over SockJS -* $SYS/# Topics -* ClientID Authentication -* IpAddress Authentication -* Username and Password Authentication -* Access control based on IpAddress, ClientID, Username -* JWT Authentication -* LDAP Authentication/ACL -* HTTP Authentication/ACL -* MySQL Authentication/ACL -* Redis Authentication/ACL -* PostgreSQL Authentication/ACL -* MongoDB Authentication/ACL -* Cluster brokers on several nodes -* Bridge brokers locally or remotely -* mosquitto, RSMB bridge -* Extensible architecture with Hooks and Plugins -* Passed eclipse paho interoperability tests -* Local Subscription -* Shared Subscription -* Proxy Protocol V1/2 -* Lua Hook and Web Hook -* LWM2M Prototol Support ## Installation -The *EMQ* broker is cross-platform, which can be deployed on Linux, Unix, Mac, Windows and even Raspberry Pi. +The *EMQ X* broker is cross-platform, which can be deployed on Linux, Unix, Mac, Windows and even Raspberry Pi. -Download the binary package for your platform from http://emqtt.io/downloads. +Download the binary package for your platform from [here](http://emqtt.io/downloads). + +- [Single Node Install](http://emqtt.io/docs/v3/install.html) +- [Multi Node Install](http://emqtt.io/docs/v3/cluster.html) -Documentation on [emqtt.io/docs/v2/](http://emqtt.io/docs/v2/install.html), [docs.emqtt.com](http://docs.emqtt.com/en/latest/install.html) for installation and configuration guide. ## Build From Source -The *EMQ* broker requires Erlang/OTP R19+ to build since 2.1 release. +The *EMQ X* broker requires Erlang/OTP R21+ to build since 3.0 release. ``` -git clone https://github.com/emqtt/emq-relx.git +git clone https://github.com/emqx/emqx-rel.git -cd emq-relx && make +cd emqx-rel && make + +cd _rel/emqx && ./bin/emqx console -cd _rel/emqttd && ./bin/emqttd console ``` -## Plugins +## Quick Start -The *EMQ* broker is highly extensible, with many hooks and plugins for customizing the authentication/ACL and integrating with other systems: + # Start emqx + ./bin/emqx start -Plugin | Description ------------------------------------------------------------------------|-------------------------------------- -[emq_plugin_template](https://github.com/emqtt/emq_plugin_template) | Plugin template and demo -[emq_dashboard](https://github.com/emqtt/emq_dashboard) | Web Dashboard -[emq_retainer](https://github.com/emqtt/emq-retainer) | Store MQTT Retained Messages -[emq_modules](https://github.com/emqtt/emq-modules) | Presence, Subscription and Rewrite Modules -[emq_auth_username](https://github.com/emqtt/emq_auth_username) | Username/Password Authentication Plugin -[emq_auth_clientid](https://github.com/emqtt/emq_auth_clientid) | ClientId Authentication Plugin -[emq_auth_mysql](https://github.com/emqtt/emq_auth_mysql) | MySQL Authentication/ACL Plugin -[emq_auth_pgsql](https://github.com/emqtt/emq_auth_pgsql) | PostgreSQL Authentication/ACL Plugin -[emq_auth_redis](https://github.com/emqtt/emq_auth_redis) | Redis Authentication/ACL Plugin -[emq_auth_mongo](https://github.com/emqtt/emq_auth_mongo) | MongoDB Authentication/ACL Plugin -[emq_auth_http](https://github.com/emqtt/emq_auth_http) | Authentication/ACL by HTTP API -[emq_auth_ldap](https://github.com/emqtt/emq_auth_ldap) | LDAP Authentication Plugin -[emq_auth_jwt](https://github.com/emqtt/emq-auth-jwt) | JWT Authentication Plugin -[emq_web_hook](https://github.com/emqtt/emq-web-hook) | Web Hook Plugin -[emq_lua_hook](https://github.com/emqtt/emq-lua-hook) | Lua Hook Plugin -[emq_sn](https://github.com/emqtt/emq_sn) | MQTT-SN Protocol Plugin -[emq_coap](https://github.com/emqtt/emq_coap) | CoAP Protocol Plugin -[emq_stomp](https://github.com/emqtt/emq_stomp) | Stomp Protocol Plugin -[emq_lwm2m](https://github.com/emqx/emqx-lwm2m) | LWM2M Prototol Plugin -[emq_recon](https://github.com/emqtt/emq_recon) | Recon Plugin -[emq_reloader](https://github.com/emqtt/emq_reloader) | Reloader Plugin -[emq_sockjs](https://github.com/emqtt/emq_sockjs) | SockJS(Stomp) Plugin + # Check Status + ./bin/emqx_ctl status -## Supports + # Stop emqx + ./bin/emqx stop -* Twitter: [@emqtt](https://twitter.com/emqtt) -* Homepage: http://emqtt.io -* Downloads: http://emqtt.io/downloads -* Documentation: http://emqtt.io/docs/v2/ -* Forum: https://groups.google.com/d/forum/emqtt -* Mailing List: -* Issues: https://github.com/emqtt/emqttd/issues -* QQ Group: 12222225 + To view the dashboard after running, use your browser to open: http://localhost:18083 -## Test Servers -The **q.emqtt.com** hosts a public Four-Node *EMQ* cluster on [QingCloud](https://qingcloud.com): +## Roadmap -![qing_cluster](http://emqtt.io/static/img/public_cluster.png) +The [EMQ X Roadmap uses Github milestones](https://github.com/emqx/emqx/milestones) to track the progress of the project. + +## Community, discussion, contribution, and support + +You can reach the EMQ community and developers via the following channels: +- [EMQX Slack](http://emqx.slack.com) + -[#emqx-users](https://emqx.slack.com/messages/CBUF2TTB8/) + -[#emqx-devs](https://emqx.slack.com/messages/CBSL57DUH/) +- [Mailing Lists]() +- [Twitter](https://twitter.com/emqtt) +- [Forum](https://groups.google.com/d/forum/emqtt) +- [Blog](https://medium.com/@emqtt) + +Please submit any bugs, issues, and feature requests to [emqx/emqx](https://github.com/emqx/emqx/issues). + +## MQTT Specifications + +You can read the mqtt protocol via the following links: + +[MQTT Version 3.1.1](https://docs.oasis-open.org/mqtt/mqtt/v3.1.1/os/mqtt-v3.1.1-os.html) + +[MQTT Version 5.0](https://docs.oasis-open.org/mqtt/mqtt/v5.0/cs02/mqtt-v5.0-cs02.html) + +[MQTT SN](http://mqtt.org/new/wp-content/uploads/2009/06/MQTT-SN_spec_v1.2.pdf) ## License -Apache License Version 2.0 +Copyright (c) 2018 [EMQ Technologies Co., Ltd](http://emqtt.io). All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License");you may not use this file except in compliance with the License.You may obtain a copy of the License at + +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and limitations under the License. diff --git a/doc/MQTT-SN_spec_v1.2.pdf b/doc/MQTT-SN_spec_v1.2.pdf deleted file mode 100644 index ee6020d23..000000000 Binary files a/doc/MQTT-SN_spec_v1.2.pdf and /dev/null differ diff --git a/doc/README b/doc/README deleted file mode 100644 index 94f96fb12..000000000 --- a/doc/README +++ /dev/null @@ -1,11 +0,0 @@ - -http://emqttd.io/docs/v2/ - -or - -http://docs.emqtt.com/ - -or - -http://emqttd-docs.rtfd.org - diff --git a/doc/mqtt-v3.1.1.pdf b/doc/mqtt-v3.1.1.pdf deleted file mode 100644 index e4095f1b5..000000000 Binary files a/doc/mqtt-v3.1.1.pdf and /dev/null differ diff --git a/erlang.mk b/erlang.mk deleted file mode 100644 index e348d4493..000000000 --- a/erlang.mk +++ /dev/null @@ -1,2741 +0,0 @@ -# Copyright (c) 2013-2015, Loïc Hoguin -# -# Permission to use, copy, modify, and/or distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -.PHONY: all app apps deps search rel docs install-docs check tests clean distclean help erlang-mk - -ERLANG_MK_FILENAME := $(realpath $(lastword $(MAKEFILE_LIST))) - -ERLANG_MK_VERSION = 2.0.0-pre.2-130-gc6fe5ea - -# Core configuration. - -PROJECT ?= $(notdir $(CURDIR)) -PROJECT := $(strip $(PROJECT)) - -PROJECT_VERSION ?= rolling -PROJECT_MOD ?= $(PROJECT)_app - -# Verbosity. - -V ?= 0 - -verbose_0 = @ -verbose_2 = set -x; -verbose = $(verbose_$(V)) - -gen_verbose_0 = @echo " GEN " $@; -gen_verbose_2 = set -x; -gen_verbose = $(gen_verbose_$(V)) - -# Temporary files directory. - -ERLANG_MK_TMP ?= $(CURDIR)/.erlang.mk -export ERLANG_MK_TMP - -# "erl" command. - -ERL = erl +A0 -noinput -boot start_clean - -# Platform detection. - -ifeq ($(PLATFORM),) -UNAME_S := $(shell uname -s) - -ifeq ($(UNAME_S),Linux) -PLATFORM = linux -else ifeq ($(UNAME_S),Darwin) -PLATFORM = darwin -else ifeq ($(UNAME_S),SunOS) -PLATFORM = solaris -else ifeq ($(UNAME_S),GNU) -PLATFORM = gnu -else ifeq ($(UNAME_S),FreeBSD) -PLATFORM = freebsd -else ifeq ($(UNAME_S),NetBSD) -PLATFORM = netbsd -else ifeq ($(UNAME_S),OpenBSD) -PLATFORM = openbsd -else ifeq ($(UNAME_S),DragonFly) -PLATFORM = dragonfly -else ifeq ($(shell uname -o),Msys) -PLATFORM = msys2 -else -$(error Unable to detect platform. Please open a ticket with the output of uname -a.) -endif - -export PLATFORM -endif - -# Core targets. - -all:: deps app rel - -# Noop to avoid a Make warning when there's nothing to do. -rel:: - $(verbose) : - -check:: tests - -clean:: clean-crashdump - -clean-crashdump: -ifneq ($(wildcard erl_crash.dump),) - $(gen_verbose) rm -f erl_crash.dump -endif - -distclean:: clean distclean-tmp - -distclean-tmp: - $(gen_verbose) rm -rf $(ERLANG_MK_TMP) - -help:: - $(verbose) printf "%s\n" \ - "erlang.mk (version $(ERLANG_MK_VERSION)) is distributed under the terms of the ISC License." \ - "Copyright (c) 2013-2015 Loïc Hoguin " \ - "" \ - "Usage: [V=1] $(MAKE) [target]..." \ - "" \ - "Core targets:" \ - " all Run deps, app and rel targets in that order" \ - " app Compile the project" \ - " deps Fetch dependencies (if needed) and compile them" \ - " search q=... Search for a package in the built-in index" \ - " rel Build a release for this project, if applicable" \ - " docs Build the documentation for this project" \ - " install-docs Install the man pages for this project" \ - " check Compile and run all tests and analysis for this project" \ - " tests Run the tests for this project" \ - " clean Delete temporary and output files from most targets" \ - " distclean Delete all temporary and output files" \ - " help Display this help and exit" \ - " erlang-mk Update erlang.mk to the latest version" - -# Core functions. - -empty := -space := $(empty) $(empty) -tab := $(empty) $(empty) -comma := , - -define newline - - -endef - -define comma_list -$(subst $(space),$(comma),$(strip $(1))) -endef - -# Adding erlang.mk to make Erlang scripts who call init:get_plain_arguments() happy. -define erlang -$(ERL) $(2) -pz $(ERLANG_MK_TMP)/rebar/ebin -eval "$(subst $(newline),,$(subst ",\",$(1)))" -- erlang.mk -endef - -ifeq ($(PLATFORM),msys2) -core_native_path = $(subst \,\\\\,$(shell cygpath -w $1)) -else -core_native_path = $1 -endif - -ifeq ($(shell which wget 2>/dev/null | wc -l), 1) -define core_http_get - wget --no-check-certificate -O $(1) $(2)|| rm $(1) -endef -else -define core_http_get.erl - ssl:start(), - inets:start(), - case httpc:request(get, {"$(2)", []}, [{autoredirect, true}], []) of - {ok, {{_, 200, _}, _, Body}} -> - case file:write_file("$(1)", Body) of - ok -> ok; - {error, R1} -> halt(R1) - end; - {error, R2} -> - halt(R2) - end, - halt(0). -endef - -define core_http_get - $(call erlang,$(call core_http_get.erl,$(call core_native_path,$1),$2)) -endef -endif - -core_eq = $(and $(findstring $(1),$(2)),$(findstring $(2),$(1))) - -core_find = $(if $(wildcard $1),$(shell find $(1:%/=%) -type f -name $(subst *,\*,$2))) - -core_lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$(1))))))))))))))))))))))))))) - -core_ls = $(filter-out $(1),$(shell echo $(1))) - -# @todo Use a solution that does not require using perl. -core_relpath = $(shell perl -e 'use File::Spec; print File::Spec->abs2rel(@ARGV) . "\n"' $1 $2) - -# Automated update. - -ERLANG_MK_REPO ?= https://github.com/ninenines/erlang.mk -ERLANG_MK_COMMIT ?= -ERLANG_MK_BUILD_CONFIG ?= build.config -ERLANG_MK_BUILD_DIR ?= .erlang.mk.build - -erlang-mk: - git clone $(ERLANG_MK_REPO) $(ERLANG_MK_BUILD_DIR) -ifdef ERLANG_MK_COMMIT - cd $(ERLANG_MK_BUILD_DIR) && git checkout $(ERLANG_MK_COMMIT) -endif - if [ -f $(ERLANG_MK_BUILD_CONFIG) ]; then cp $(ERLANG_MK_BUILD_CONFIG) $(ERLANG_MK_BUILD_DIR)/build.config; fi - $(MAKE) -C $(ERLANG_MK_BUILD_DIR) - cp $(ERLANG_MK_BUILD_DIR)/erlang.mk ./erlang.mk - rm -rf $(ERLANG_MK_BUILD_DIR) - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: search - -define pkg_print - $(verbose) printf "%s\n" \ - $(if $(call core_eq,$(1),$(pkg_$(1)_name)),,"Pkg name: $(1)") \ - "App name: $(pkg_$(1)_name)" \ - "Description: $(pkg_$(1)_description)" \ - "Home page: $(pkg_$(1)_homepage)" \ - "Fetch with: $(pkg_$(1)_fetch)" \ - "Repository: $(pkg_$(1)_repo)" \ - "Commit: $(pkg_$(1)_commit)" \ - "" - -endef - -search: -ifdef q - $(foreach p,$(PACKAGES), \ - $(if $(findstring $(call core_lc,$(q)),$(call core_lc,$(pkg_$(p)_name) $(pkg_$(p)_description))), \ - $(call pkg_print,$(p)))) -else - $(foreach p,$(PACKAGES),$(call pkg_print,$(p))) -endif - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: distclean-deps - -# Configuration. - -ifdef OTP_DEPS -$(warning The variable OTP_DEPS is deprecated in favor of LOCAL_DEPS.) -endif - -IGNORE_DEPS ?= -export IGNORE_DEPS - -APPS_DIR ?= $(CURDIR)/apps -export APPS_DIR - -DEPS_DIR ?= $(CURDIR)/deps -export DEPS_DIR - -REBAR_DEPS_DIR = $(DEPS_DIR) -export REBAR_DEPS_DIR - -dep_name = $(if $(dep_$(1)),$(1),$(if $(pkg_$(1)_name),$(pkg_$(1)_name),$(1))) -dep_repo = $(patsubst git://github.com/%,https://github.com/%, \ - $(if $(dep_$(1)),$(word 2,$(dep_$(1))),$(pkg_$(1)_repo))) -dep_commit = $(if $(dep_$(1)_commit),$(dep_$(1)_commit),$(if $(dep_$(1)),$(word 3,$(dep_$(1))),$(pkg_$(1)_commit))) - -ALL_APPS_DIRS = $(if $(wildcard $(APPS_DIR)/),$(filter-out $(APPS_DIR),$(shell find $(APPS_DIR) -maxdepth 1 -type d))) -ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call dep_name,$(dep)))) - -ifeq ($(filter $(APPS_DIR) $(DEPS_DIR),$(subst :, ,$(ERL_LIBS))),) -ifeq ($(ERL_LIBS),) - ERL_LIBS = $(APPS_DIR):$(DEPS_DIR) -else - ERL_LIBS := $(ERL_LIBS):$(APPS_DIR):$(DEPS_DIR) -endif -endif -export ERL_LIBS - -export NO_AUTOPATCH - -# Verbosity. - -dep_verbose_0 = @echo " DEP " $(1); -dep_verbose_2 = set -x; -dep_verbose = $(dep_verbose_$(V)) - -# Core targets. - -ifdef IS_APP -apps:: -else -apps:: $(ALL_APPS_DIRS) -ifeq ($(IS_APP)$(IS_DEP),) - $(verbose) rm -f $(ERLANG_MK_TMP)/apps.log -endif - $(verbose) mkdir -p $(ERLANG_MK_TMP) -# Create ebin directory for all apps to make sure Erlang recognizes them -# as proper OTP applications when using -include_lib. This is a temporary -# fix, a proper fix would be to compile apps/* in the right order. - $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ - mkdir -p $$dep/ebin || exit $$?; \ - done - $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ - if grep -qs ^$$dep$$ $(ERLANG_MK_TMP)/apps.log; then \ - :; \ - else \ - echo $$dep >> $(ERLANG_MK_TMP)/apps.log; \ - $(MAKE) -C $$dep IS_APP=1 || exit $$?; \ - fi \ - done -endif - -ifneq ($(SKIP_DEPS),) -deps:: -else -deps:: $(ALL_DEPS_DIRS) apps -ifeq ($(IS_APP)$(IS_DEP),) - $(verbose) rm -f $(ERLANG_MK_TMP)/deps.log -endif - $(verbose) mkdir -p $(ERLANG_MK_TMP) - $(verbose) for dep in $(ALL_DEPS_DIRS) ; do \ - if grep -qs ^$$dep$$ $(ERLANG_MK_TMP)/deps.log; then \ - :; \ - else \ - echo $$dep >> $(ERLANG_MK_TMP)/deps.log; \ - if [ -f $$dep/GNUmakefile ] || [ -f $$dep/makefile ] || [ -f $$dep/Makefile ]; then \ - $(MAKE) -C $$dep IS_DEP=1 || exit $$?; \ - else \ - echo "Error: No Makefile to build dependency $$dep."; \ - exit 2; \ - fi \ - fi \ - done -endif - -# Deps related targets. - -# @todo rename GNUmakefile and makefile into Makefile first, if they exist -# While Makefile file could be GNUmakefile or makefile, -# in practice only Makefile is needed so far. -define dep_autopatch - if [ -f $(DEPS_DIR)/$(1)/erlang.mk ]; then \ - $(call erlang,$(call dep_autopatch_appsrc.erl,$(1))); \ - $(call dep_autopatch_erlang_mk,$(1)); \ - elif [ -f $(DEPS_DIR)/$(1)/Makefile ]; then \ - if [ 0 != `grep -c "include ../\w*\.mk" $(DEPS_DIR)/$(1)/Makefile` ]; then \ - $(call dep_autopatch2,$(1)); \ - elif [ 0 != `grep -ci rebar $(DEPS_DIR)/$(1)/Makefile` ]; then \ - $(call dep_autopatch2,$(1)); \ - elif [ -n "`find $(DEPS_DIR)/$(1)/ -type f -name \*.mk -not -name erlang.mk -exec grep -i rebar '{}' \;`" ]; then \ - $(call dep_autopatch2,$(1)); \ - else \ - $(call erlang,$(call dep_autopatch_app.erl,$(1))); \ - fi \ - else \ - if [ ! -d $(DEPS_DIR)/$(1)/src/ ]; then \ - $(call dep_autopatch_noop,$(1)); \ - else \ - $(call dep_autopatch2,$(1)); \ - fi \ - fi -endef - -define dep_autopatch2 - if [ -f $(DEPS_DIR)/$1/src/$1.app.src.script ]; then \ - $(call erlang,$(call dep_autopatch_appsrc_script.erl,$(1))); \ - fi; \ - $(call erlang,$(call dep_autopatch_appsrc.erl,$(1))); \ - if [ -f $(DEPS_DIR)/$(1)/rebar -o -f $(DEPS_DIR)/$(1)/rebar.config -o -f $(DEPS_DIR)/$(1)/rebar.config.script ]; then \ - $(call dep_autopatch_fetch_rebar); \ - $(call dep_autopatch_rebar,$(1)); \ - else \ - $(call dep_autopatch_gen,$(1)); \ - fi -endef - -define dep_autopatch_noop - printf "noop:\n" > $(DEPS_DIR)/$(1)/Makefile -endef - -# Overwrite erlang.mk with the current file by default. -ifeq ($(NO_AUTOPATCH_ERLANG_MK),) -define dep_autopatch_erlang_mk - echo "include $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(DEPS_DIR)/app)/erlang.mk" \ - > $(DEPS_DIR)/$1/erlang.mk -endef -else -define dep_autopatch_erlang_mk - : -endef -endif - -define dep_autopatch_gen - printf "%s\n" \ - "ERLC_OPTS = +debug_info" \ - "include ../../erlang.mk" > $(DEPS_DIR)/$(1)/Makefile -endef - -define dep_autopatch_fetch_rebar - mkdir -p $(ERLANG_MK_TMP); \ - if [ ! -d $(ERLANG_MK_TMP)/rebar ]; then \ - git clone -q -n -- https://github.com/rebar/rebar $(ERLANG_MK_TMP)/rebar; \ - cd $(ERLANG_MK_TMP)/rebar; \ - git checkout -q 791db716b5a3a7671e0b351f95ddf24b848ee173; \ - $(MAKE); \ - cd -; \ - fi -endef - -define dep_autopatch_rebar - if [ -f $(DEPS_DIR)/$(1)/Makefile ]; then \ - mv $(DEPS_DIR)/$(1)/Makefile $(DEPS_DIR)/$(1)/Makefile.orig.mk; \ - fi; \ - $(call erlang,$(call dep_autopatch_rebar.erl,$(1))); \ - rm -f $(DEPS_DIR)/$(1)/ebin/$(1).app -endef - -define dep_autopatch_rebar.erl - application:load(rebar), - application:set_env(rebar, log_level, debug), - Conf1 = case file:consult("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config)") of - {ok, Conf0} -> Conf0; - _ -> [] - end, - {Conf, OsEnv} = fun() -> - case filelib:is_file("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)") of - false -> {Conf1, []}; - true -> - Bindings0 = erl_eval:new_bindings(), - Bindings1 = erl_eval:add_binding('CONFIG', Conf1, Bindings0), - Bindings = erl_eval:add_binding('SCRIPT', "$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)", Bindings1), - Before = os:getenv(), - {ok, Conf2} = file:script("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)", Bindings), - {Conf2, lists:foldl(fun(E, Acc) -> lists:delete(E, Acc) end, os:getenv(), Before)} - end - end(), - Write = fun (Text) -> - file:write_file("$(call core_native_path,$(DEPS_DIR)/$1/Makefile)", Text, [append]) - end, - Escape = fun (Text) -> - re:replace(Text, "\\\\$$", "\$$$$", [global, {return, list}]) - end, - Write("IGNORE_DEPS += edown eper eunit_formatters meck node_package " - "rebar_lock_deps_plugin rebar_vsn_plugin reltool_util\n"), - Write("C_SRC_DIR = /path/do/not/exist\n"), - Write("C_SRC_TYPE = rebar\n"), - Write("DRV_CFLAGS = -fPIC\nexport DRV_CFLAGS\n"), - Write(["ERLANG_ARCH = ", rebar_utils:wordsize(), "\nexport ERLANG_ARCH\n"]), - fun() -> - Write("ERLC_OPTS = +debug_info\nexport ERLC_OPTS\n"), - case lists:keyfind(erl_opts, 1, Conf) of - false -> ok; - {_, ErlOpts} -> - lists:foreach(fun - ({d, D}) -> - Write("ERLC_OPTS += -D" ++ atom_to_list(D) ++ "=1\n"); - ({i, I}) -> - Write(["ERLC_OPTS += -I ", I, "\n"]); - ({platform_define, Regex, D}) -> - case rebar_utils:is_arch(Regex) of - true -> Write("ERLC_OPTS += -D" ++ atom_to_list(D) ++ "=1\n"); - false -> ok - end; - ({parse_transform, PT}) -> - Write("ERLC_OPTS += +'{parse_transform, " ++ atom_to_list(PT) ++ "}'\n"); - (_) -> ok - end, ErlOpts) - end, - Write("\n") - end(), - fun() -> - File = case lists:keyfind(deps, 1, Conf) of - false -> []; - {_, Deps} -> - [begin case case Dep of - {N, S} when is_atom(N), is_list(S) -> {N, {hex, S}}; - {N, S} when is_tuple(S) -> {N, S}; - {N, _, S} -> {N, S}; - {N, _, S, _} -> {N, S}; - _ -> false - end of - false -> ok; - {Name, Source} -> - {Method, Repo, Commit} = case Source of - {hex, V} -> {hex, V, undefined}; - {git, R} -> {git, R, master}; - {M, R, {branch, C}} -> {M, R, C}; - {M, R, {ref, C}} -> {M, R, C}; - {M, R, {tag, C}} -> {M, R, C}; - {M, R, C} -> {M, R, C} - end, - Write(io_lib:format("DEPS += ~s\ndep_~s = ~s ~s ~s~n", [Name, Name, Method, Repo, Commit])) - end end || Dep <- Deps] - end - end(), - fun() -> - case lists:keyfind(erl_first_files, 1, Conf) of - false -> ok; - {_, Files} -> - Names = [[" ", case lists:reverse(F) of - "lre." ++ Elif -> lists:reverse(Elif); - Elif -> lists:reverse(Elif) - end] || "src/" ++ F <- Files], - Write(io_lib:format("COMPILE_FIRST +=~s\n", [Names])) - end - end(), - Write("\n\nrebar_dep: preprocess pre-deps deps pre-app app\n"), - Write("\npreprocess::\n"), - Write("\npre-deps::\n"), - Write("\npre-app::\n"), - PatchHook = fun(Cmd) -> - case Cmd of - "make -C" ++ Cmd1 -> "$$\(MAKE) -C" ++ Escape(Cmd1); - "gmake -C" ++ Cmd1 -> "$$\(MAKE) -C" ++ Escape(Cmd1); - "make " ++ Cmd1 -> "$$\(MAKE) -f Makefile.orig.mk " ++ Escape(Cmd1); - "gmake " ++ Cmd1 -> "$$\(MAKE) -f Makefile.orig.mk " ++ Escape(Cmd1); - _ -> Escape(Cmd) - end - end, - fun() -> - case lists:keyfind(pre_hooks, 1, Conf) of - false -> ok; - {_, Hooks} -> - [case H of - {'get-deps', Cmd} -> - Write("\npre-deps::\n\t" ++ PatchHook(Cmd) ++ "\n"); - {compile, Cmd} -> - Write("\npre-app::\n\tCC=$$\(CC) " ++ PatchHook(Cmd) ++ "\n"); - {Regex, compile, Cmd} -> - case rebar_utils:is_arch(Regex) of - true -> Write("\npre-app::\n\tCC=$$\(CC) " ++ PatchHook(Cmd) ++ "\n"); - false -> ok - end; - _ -> ok - end || H <- Hooks] - end - end(), - ShellToMk = fun(V) -> - re:replace(re:replace(V, "(\\\\$$)(\\\\w*)", "\\\\1(\\\\2)", [global]), - "-Werror\\\\b", "", [{return, list}, global]) - end, - PortSpecs = fun() -> - case lists:keyfind(port_specs, 1, Conf) of - false -> - case filelib:is_dir("$(call core_native_path,$(DEPS_DIR)/$1/c_src)") of - false -> []; - true -> - [{"priv/" ++ proplists:get_value(so_name, Conf, "$(1)_drv.so"), - proplists:get_value(port_sources, Conf, ["c_src/*.c"]), []}] - end; - {_, Specs} -> - lists:flatten([case S of - {Output, Input} -> {ShellToMk(Output), Input, []}; - {Regex, Output, Input} -> - case rebar_utils:is_arch(Regex) of - true -> {ShellToMk(Output), Input, []}; - false -> [] - end; - {Regex, Output, Input, [{env, Env}]} -> - case rebar_utils:is_arch(Regex) of - true -> {ShellToMk(Output), Input, Env}; - false -> [] - end - end || S <- Specs]) - end - end(), - PortSpecWrite = fun (Text) -> - file:write_file("$(call core_native_path,$(DEPS_DIR)/$1/c_src/Makefile.erlang.mk)", Text, [append]) - end, - case PortSpecs of - [] -> ok; - _ -> - Write("\npre-app::\n\t$$\(MAKE) -f c_src/Makefile.erlang.mk\n"), - PortSpecWrite(io_lib:format("ERL_CFLAGS = -finline-functions -Wall -fPIC -I \\"~s/erts-~s/include\\" -I \\"~s\\"\n", - [code:root_dir(), erlang:system_info(version), code:lib_dir(erl_interface, include)])), - PortSpecWrite(io_lib:format("ERL_LDFLAGS = -L \\"~s\\" -lerl_interface -lei\n", - [code:lib_dir(erl_interface, lib)])), - [PortSpecWrite(["\n", E, "\n"]) || E <- OsEnv], - FilterEnv = fun(Env) -> - lists:flatten([case E of - {_, _} -> E; - {Regex, K, V} -> - case rebar_utils:is_arch(Regex) of - true -> {K, V}; - false -> [] - end - end || E <- Env]) - end, - MergeEnv = fun(Env) -> - lists:foldl(fun ({K, V}, Acc) -> - case lists:keyfind(K, 1, Acc) of - false -> [{K, rebar_utils:expand_env_variable(V, K, "")}|Acc]; - {_, V0} -> [{K, rebar_utils:expand_env_variable(V, K, V0)}|Acc] - end - end, [], Env) - end, - PortEnv = case lists:keyfind(port_env, 1, Conf) of - false -> []; - {_, PortEnv0} -> FilterEnv(PortEnv0) - end, - PortSpec = fun ({Output, Input0, Env}) -> - filelib:ensure_dir("$(call core_native_path,$(DEPS_DIR)/$1/)" ++ Output), - Input = [[" ", I] || I <- Input0], - PortSpecWrite([ - [["\n", K, " = ", ShellToMk(V)] || {K, V} <- lists:reverse(MergeEnv(PortEnv))], - case $(PLATFORM) of - darwin -> "\n\nLDFLAGS += -flat_namespace -undefined suppress"; - _ -> "" - end, - "\n\nall:: ", Output, "\n\n", - "%.o: %.c\n\t$$\(CC) -c -o $$\@ $$\< $$\(CFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - "%.o: %.C\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - "%.o: %.cc\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - "%.o: %.cpp\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", - [[Output, ": ", K, " = ", ShellToMk(V), "\n"] || {K, V} <- lists:reverse(MergeEnv(FilterEnv(Env)))], - Output, ": $$\(foreach ext,.c .C .cc .cpp,", - "$$\(patsubst %$$\(ext),%.o,$$\(filter %$$\(ext),$$\(wildcard", Input, "))))\n", - "\t$$\(CC) -o $$\@ $$\? $$\(LDFLAGS) $$\(ERL_LDFLAGS) $$\(DRV_LDFLAGS) $$\(EXE_LDFLAGS)", - case {filename:extension(Output), $(PLATFORM)} of - {[], _} -> "\n"; - {_, darwin} -> "\n"; - _ -> " -shared\n" - end]) - end, - [PortSpec(S) || S <- PortSpecs] - end, - Write("\ninclude $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(DEPS_DIR)/app)/erlang.mk"), - RunPlugin = fun(Plugin, Step) -> - case erlang:function_exported(Plugin, Step, 2) of - false -> ok; - true -> - c:cd("$(call core_native_path,$(DEPS_DIR)/$1/)"), - Ret = Plugin:Step({config, "", Conf, dict:new(), dict:new(), dict:new(), - dict:store(base_dir, "", dict:new())}, undefined), - io:format("rebar plugin ~p step ~p ret ~p~n", [Plugin, Step, Ret]) - end - end, - fun() -> - case lists:keyfind(plugins, 1, Conf) of - false -> ok; - {_, Plugins} -> - [begin - case lists:keyfind(deps, 1, Conf) of - false -> ok; - {_, Deps} -> - case lists:keyfind(P, 1, Deps) of - false -> ok; - _ -> - Path = "$(call core_native_path,$(DEPS_DIR)/)" ++ atom_to_list(P), - io:format("~s", [os:cmd("$(MAKE) -C $(call core_native_path,$(DEPS_DIR)/$1) " ++ Path)]), - io:format("~s", [os:cmd("$(MAKE) -C " ++ Path ++ " IS_DEP=1")]), - code:add_patha(Path ++ "/ebin") - end - end - end || P <- Plugins], - [case code:load_file(P) of - {module, P} -> ok; - _ -> - case lists:keyfind(plugin_dir, 1, Conf) of - false -> ok; - {_, PluginsDir} -> - ErlFile = "$(call core_native_path,$(DEPS_DIR)/$1/)" ++ PluginsDir ++ "/" ++ atom_to_list(P) ++ ".erl", - {ok, P, Bin} = compile:file(ErlFile, [binary]), - {module, P} = code:load_binary(P, ErlFile, Bin) - end - end || P <- Plugins], - [RunPlugin(P, preprocess) || P <- Plugins], - [RunPlugin(P, pre_compile) || P <- Plugins], - [RunPlugin(P, compile) || P <- Plugins] - end - end(), - halt() -endef - -define dep_autopatch_app.erl - UpdateModules = fun(App) -> - case filelib:is_regular(App) of - false -> ok; - true -> - {ok, [{application, '$(1)', L0}]} = file:consult(App), - Mods = filelib:fold_files("$(call core_native_path,$(DEPS_DIR)/$1/src)", "\\\\.erl$$", true, - fun (F, Acc) -> [list_to_atom(filename:rootname(filename:basename(F)))|Acc] end, []), - L = lists:keystore(modules, 1, L0, {modules, Mods}), - ok = file:write_file(App, io_lib:format("~p.~n", [{application, '$(1)', L}])) - end - end, - UpdateModules("$(call core_native_path,$(DEPS_DIR)/$1/ebin/$1.app)"), - halt() -endef - -define dep_autopatch_appsrc_script.erl - AppSrc = "$(call core_native_path,$(DEPS_DIR)/$1/src/$1.app.src)", - AppSrcScript = AppSrc ++ ".script", - Bindings = erl_eval:new_bindings(), - {ok, Conf} = file:script(AppSrcScript, Bindings), - ok = file:write_file(AppSrc, io_lib:format("~p.~n", [Conf])), - halt() -endef - -define dep_autopatch_appsrc.erl - AppSrcOut = "$(call core_native_path,$(DEPS_DIR)/$1/src/$1.app.src)", - AppSrcIn = case filelib:is_regular(AppSrcOut) of false -> "$(call core_native_path,$(DEPS_DIR)/$1/ebin/$1.app)"; true -> AppSrcOut end, - case filelib:is_regular(AppSrcIn) of - false -> ok; - true -> - {ok, [{application, $(1), L0}]} = file:consult(AppSrcIn), - L1 = lists:keystore(modules, 1, L0, {modules, []}), - L2 = case lists:keyfind(vsn, 1, L1) of {_, git} -> lists:keyreplace(vsn, 1, L1, {vsn, "git"}); _ -> L1 end, - L3 = case lists:keyfind(registered, 1, L2) of false -> [{registered, []}|L2]; _ -> L2 end, - ok = file:write_file(AppSrcOut, io_lib:format("~p.~n", [{application, $(1), L3}])), - case AppSrcOut of AppSrcIn -> ok; _ -> ok = file:delete(AppSrcIn) end - end, - halt() -endef - -define dep_fetch_git - git clone -q -n -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \ - cd $(DEPS_DIR)/$(call dep_name,$(1)) && git checkout -q $(call dep_commit,$(1)); -endef - -define dep_fetch_git-submodule - git submodule update --init -- $(DEPS_DIR)/$1; -endef - -define dep_fetch_hg - hg clone -q -U $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \ - cd $(DEPS_DIR)/$(call dep_name,$(1)) && hg update -q $(call dep_commit,$(1)); -endef - -define dep_fetch_svn - svn checkout -q $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); -endef - -define dep_fetch_cp - cp -R $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); -endef - -define dep_fetch_hex.erl - ssl:start(), - inets:start(), - {ok, {{_, 200, _}, _, Body}} = httpc:request(get, - {"https://s3.amazonaws.com/s3.hex.pm/tarballs/$(1)-$(2).tar", []}, - [], [{body_format, binary}]), - {ok, Files} = erl_tar:extract({binary, Body}, [memory]), - {_, Source} = lists:keyfind("contents.tar.gz", 1, Files), - ok = erl_tar:extract({binary, Source}, [{cwd, "$(call core_native_path,$(DEPS_DIR)/$1)"}, compressed]), - halt() -endef - -# Hex only has a package version. No need to look in the Erlang.mk packages. -define dep_fetch_hex - $(call erlang,$(call dep_fetch_hex.erl,$(1),$(strip $(word 2,$(dep_$(1)))))); -endef - -define dep_fetch_fail - echo "Error: Unknown or invalid dependency: $(1)." >&2; \ - exit 78; -endef - -# Kept for compatibility purposes with older Erlang.mk configuration. -define dep_fetch_legacy - $(warning WARNING: '$(1)' dependency configuration uses deprecated format.) \ - git clone -q -n -- $(word 1,$(dep_$(1))) $(DEPS_DIR)/$(1); \ - cd $(DEPS_DIR)/$(1) && git checkout -q $(if $(word 2,$(dep_$(1))),$(word 2,$(dep_$(1))),master); -endef - -define dep_fetch - $(if $(dep_$(1)), \ - $(if $(dep_fetch_$(word 1,$(dep_$(1)))), \ - $(word 1,$(dep_$(1))), \ - $(if $(IS_DEP),legacy,fail)), \ - $(if $(filter $(1),$(PACKAGES)), \ - $(pkg_$(1)_fetch), \ - fail)) -endef - -define dep_target -$(DEPS_DIR)/$(call dep_name,$1): - $(eval DEP_NAME := $(call dep_name,$1)) - $(eval DEP_STR := $(if $(filter-out $1,$(DEP_NAME)),$1,"$1 ($(DEP_NAME))")) - $(verbose) if test -d $(APPS_DIR)/$(DEP_NAME); then \ - echo "Error: Dependency" $(DEP_STR) "conflicts with application found in $(APPS_DIR)/$(DEP_NAME)."; \ - exit 17; \ - fi - $(verbose) mkdir -p $(DEPS_DIR) - $(dep_verbose) $(call dep_fetch_$(strip $(call dep_fetch,$(1))),$(1)) - $(verbose) if [ -f $(DEPS_DIR)/$(1)/configure.ac -o -f $(DEPS_DIR)/$(1)/configure.in ] \ - && [ ! -f $(DEPS_DIR)/$(1)/configure ]; then \ - echo " AUTO " $(1); \ - cd $(DEPS_DIR)/$(1) && autoreconf -Wall -vif -I m4; \ - fi - - $(verbose) if [ -f $(DEPS_DIR)/$(DEP_NAME)/configure ]; then \ - echo " CONF " $(DEP_STR); \ - cd $(DEPS_DIR)/$(DEP_NAME) && ./configure; \ - fi -ifeq ($(filter $(1),$(NO_AUTOPATCH)),) - $(verbose) if [ "$(1)" = "amqp_client" -a "$(RABBITMQ_CLIENT_PATCH)" ]; then \ - if [ ! -d $(DEPS_DIR)/rabbitmq-codegen ]; then \ - echo " PATCH Downloading rabbitmq-codegen"; \ - git clone https://github.com/rabbitmq/rabbitmq-codegen.git $(DEPS_DIR)/rabbitmq-codegen; \ - fi; \ - if [ ! -d $(DEPS_DIR)/rabbitmq-server ]; then \ - echo " PATCH Downloading rabbitmq-server"; \ - git clone https://github.com/rabbitmq/rabbitmq-server.git $(DEPS_DIR)/rabbitmq-server; \ - fi; \ - ln -s $(DEPS_DIR)/amqp_client/deps/rabbit_common-0.0.0 $(DEPS_DIR)/rabbit_common; \ - elif [ "$(1)" = "rabbit" -a "$(RABBITMQ_SERVER_PATCH)" ]; then \ - if [ ! -d $(DEPS_DIR)/rabbitmq-codegen ]; then \ - echo " PATCH Downloading rabbitmq-codegen"; \ - git clone https://github.com/rabbitmq/rabbitmq-codegen.git $(DEPS_DIR)/rabbitmq-codegen; \ - fi \ - else \ - $$(call dep_autopatch,$(DEP_NAME)) \ - fi -endif -endef - -$(foreach dep,$(BUILD_DEPS) $(DEPS),$(eval $(call dep_target,$(dep)))) - -ifndef IS_APP -clean:: clean-apps - -clean-apps: - $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ - $(MAKE) -C $$dep clean IS_APP=1 || exit $$?; \ - done - -distclean:: distclean-apps - -distclean-apps: - $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ - $(MAKE) -C $$dep distclean IS_APP=1 || exit $$?; \ - done -endif - -ifndef SKIP_DEPS -distclean:: distclean-deps - -distclean-deps: - $(gen_verbose) rm -rf $(DEPS_DIR) -endif - -# External plugins. - -DEP_PLUGINS ?= - -define core_dep_plugin --include $(DEPS_DIR)/$(1) - -$(DEPS_DIR)/$(1): $(DEPS_DIR)/$(2) ; -endef - -$(foreach p,$(DEP_PLUGINS),\ - $(eval $(if $(findstring /,$p),\ - $(call core_dep_plugin,$p,$(firstword $(subst /, ,$p))),\ - $(call core_dep_plugin,$p/plugins.mk,$p)))) - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -# Configuration. - -DTL_FULL_PATH ?= -DTL_PATH ?= templates/ -DTL_SUFFIX ?= _dtl -DTL_OPTS ?= - -# Verbosity. - -dtl_verbose_0 = @echo " DTL " $(filter %.dtl,$(?F)); -dtl_verbose = $(dtl_verbose_$(V)) - -# Core targets. - -DTL_FILES = $(sort $(call core_find,$(DTL_PATH),*.dtl)) - -ifneq ($(DTL_FILES),) - -ifdef DTL_FULL_PATH -BEAM_FILES += $(addprefix ebin/,$(patsubst %.dtl,%_dtl.beam,$(subst /,_,$(DTL_FILES:$(DTL_PATH)%=%)))) -else -BEAM_FILES += $(addprefix ebin/,$(patsubst %.dtl,%_dtl.beam,$(notdir $(DTL_FILES)))) -endif - -# Rebuild templates when the Makefile changes. -$(DTL_FILES): $(MAKEFILE_LIST) - @touch $@ - -define erlydtl_compile.erl - [begin - Module0 = case "$(strip $(DTL_FULL_PATH))" of - "" -> - filename:basename(F, ".dtl"); - _ -> - "$(DTL_PATH)" ++ F2 = filename:rootname(F, ".dtl"), - re:replace(F2, "/", "_", [{return, list}, global]) - end, - Module = list_to_atom(string:to_lower(Module0) ++ "$(DTL_SUFFIX)"), - case erlydtl:compile(F, Module, [$(DTL_OPTS)] ++ [{out_dir, "ebin/"}, return_errors, {doc_root, "templates"}]) of - ok -> ok; - {ok, _} -> ok - end - end || F <- string:tokens("$(1)", " ")], - halt(). -endef - -ebin/$(PROJECT).app:: $(DTL_FILES) | ebin/ - $(if $(strip $?),\ - $(dtl_verbose) $(call erlang,$(call erlydtl_compile.erl,$?),-pa ebin/ $(DEPS_DIR)/erlydtl/ebin/)) - -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -# Verbosity. - -proto_verbose_0 = @echo " PROTO " $(filter %.proto,$(?F)); -proto_verbose = $(proto_verbose_$(V)) - -# Core targets. - -define compile_proto - $(verbose) mkdir -p ebin/ include/ - $(proto_verbose) $(call erlang,$(call compile_proto.erl,$(1))) - $(proto_verbose) erlc +debug_info -o ebin/ ebin/*.erl - $(verbose) rm ebin/*.erl -endef - -define compile_proto.erl - [begin - Dir = filename:dirname(filename:dirname(F)), - protobuffs_compile:generate_source(F, - [{output_include_dir, Dir ++ "/include"}, - {output_src_dir, Dir ++ "/ebin"}]) - end || F <- string:tokens("$(1)", " ")], - halt(). -endef - -ifneq ($(wildcard src/),) -ebin/$(PROJECT).app:: $(sort $(call core_find,src/,*.proto)) - $(if $(strip $?),$(call compile_proto,$?)) -endif - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: clean-app - -# Configuration. - -ERLC_OPTS ?= -Werror +debug_info +warn_export_vars +warn_shadow_vars \ - +warn_obsolete_guard # +bin_opt_info +warn_export_all +warn_missing_spec -COMPILE_FIRST ?= -COMPILE_FIRST_PATHS = $(addprefix src/,$(addsuffix .erl,$(COMPILE_FIRST))) -ERLC_EXCLUDE ?= -ERLC_EXCLUDE_PATHS = $(addprefix src/,$(addsuffix .erl,$(ERLC_EXCLUDE))) - -ERLC_MIB_OPTS ?= -COMPILE_MIB_FIRST ?= -COMPILE_MIB_FIRST_PATHS = $(addprefix mibs/,$(addsuffix .mib,$(COMPILE_MIB_FIRST))) - -# Verbosity. - -app_verbose_0 = @echo " APP " $(PROJECT); -app_verbose_2 = set -x; -app_verbose = $(app_verbose_$(V)) - -appsrc_verbose_0 = @echo " APP " $(PROJECT).app.src; -appsrc_verbose_2 = set -x; -appsrc_verbose = $(appsrc_verbose_$(V)) - -makedep_verbose_0 = @echo " DEPEND" $(PROJECT).d; -makedep_verbose_2 = set -x; -makedep_verbose = $(makedep_verbose_$(V)) - -erlc_verbose_0 = @echo " ERLC " $(filter-out $(patsubst %,%.erl,$(ERLC_EXCLUDE)),\ - $(filter %.erl %.core,$(?F))); -erlc_verbose_2 = set -x; -erlc_verbose = $(erlc_verbose_$(V)) - -xyrl_verbose_0 = @echo " XYRL " $(filter %.xrl %.yrl,$(?F)); -xyrl_verbose_2 = set -x; -xyrl_verbose = $(xyrl_verbose_$(V)) - -asn1_verbose_0 = @echo " ASN1 " $(filter %.asn1,$(?F)); -asn1_verbose_2 = set -x; -asn1_verbose = $(asn1_verbose_$(V)) - -mib_verbose_0 = @echo " MIB " $(filter %.bin %.mib,$(?F)); -mib_verbose_2 = set -x; -mib_verbose = $(mib_verbose_$(V)) - -ifneq ($(wildcard src/),) - -# Targets. - -ifeq ($(wildcard ebin/test),) -app:: deps $(PROJECT).d - $(verbose) $(MAKE) --no-print-directory app-build -else -app:: clean deps $(PROJECT).d - $(verbose) $(MAKE) --no-print-directory app-build -endif - -ifeq ($(wildcard src/$(PROJECT_MOD).erl),) -define app_file -{application, $(PROJECT), [ - {description, "$(PROJECT_DESCRIPTION)"}, - {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), - {id$(comma)$(space)"$(1)"}$(comma)) - {modules, [$(call comma_list,$(2))]}, - {registered, []}, - {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]} -]}. -endef -else -define app_file -{application, $(PROJECT), [ - {description, "$(PROJECT_DESCRIPTION)"}, - {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), - {id$(comma)$(space)"$(1)"}$(comma)) - {modules, [$(call comma_list,$(2))]}, - {registered, [$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED))]}, - {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]}, - {mod, {$(PROJECT_MOD), []}} -]}. -endef -endif - -app-build: ebin/$(PROJECT).app - $(verbose) : - -# Source files. - -ERL_FILES = $(sort $(call core_find,src/,*.erl)) -CORE_FILES = $(sort $(call core_find,src/,*.core)) - -# ASN.1 files. - -ifneq ($(wildcard asn1/),) -ASN1_FILES = $(sort $(call core_find,asn1/,*.asn1)) -ERL_FILES += $(addprefix src/,$(patsubst %.asn1,%.erl,$(notdir $(ASN1_FILES)))) - -define compile_asn1 - $(verbose) mkdir -p include/ - $(asn1_verbose) erlc -v -I include/ -o asn1/ +noobj $(1) - $(verbose) mv asn1/*.erl src/ - $(verbose) mv asn1/*.hrl include/ - $(verbose) mv asn1/*.asn1db include/ -endef - -$(PROJECT).d:: $(ASN1_FILES) - $(if $(strip $?),$(call compile_asn1,$?)) -endif - -# SNMP MIB files. - -ifneq ($(wildcard mibs/),) -MIB_FILES = $(sort $(call core_find,mibs/,*.mib)) - -$(PROJECT).d:: $(COMPILE_MIB_FIRST_PATHS) $(MIB_FILES) - $(verbose) mkdir -p include/ priv/mibs/ - $(mib_verbose) erlc -v $(ERLC_MIB_OPTS) -o priv/mibs/ -I priv/mibs/ $? - $(mib_verbose) erlc -o include/ -- $(addprefix priv/mibs/,$(patsubst %.mib,%.bin,$(notdir $?))) -endif - -# Leex and Yecc files. - -XRL_FILES = $(sort $(call core_find,src/,*.xrl)) -XRL_ERL_FILES = $(addprefix src/,$(patsubst %.xrl,%.erl,$(notdir $(XRL_FILES)))) -ERL_FILES += $(XRL_ERL_FILES) - -YRL_FILES = $(sort $(call core_find,src/,*.yrl)) -YRL_ERL_FILES = $(addprefix src/,$(patsubst %.yrl,%.erl,$(notdir $(YRL_FILES)))) -ERL_FILES += $(YRL_ERL_FILES) - -$(PROJECT).d:: $(XRL_FILES) $(YRL_FILES) - $(if $(strip $?),$(xyrl_verbose) erlc -v -o src/ $?) - -# Erlang and Core Erlang files. - -define makedep.erl - E = ets:new(makedep, [bag]), - G = digraph:new([acyclic]), - ErlFiles = lists:usort(string:tokens("$(ERL_FILES)", " ")), - Modules = [{list_to_atom(filename:basename(F, ".erl")), F} || F <- ErlFiles], - Add = fun (Mod, Dep) -> - case lists:keyfind(Dep, 1, Modules) of - false -> ok; - {_, DepFile} -> - {_, ModFile} = lists:keyfind(Mod, 1, Modules), - ets:insert(E, {ModFile, DepFile}), - digraph:add_vertex(G, Mod), - digraph:add_vertex(G, Dep), - digraph:add_edge(G, Mod, Dep) - end - end, - AddHd = fun (F, Mod, DepFile) -> - case file:open(DepFile, [read]) of - {error, enoent} -> ok; - {ok, Fd} -> - F(F, Fd, Mod), - {_, ModFile} = lists:keyfind(Mod, 1, Modules), - ets:insert(E, {ModFile, DepFile}) - end - end, - Attr = fun - (F, Mod, behavior, Dep) -> Add(Mod, Dep); - (F, Mod, behaviour, Dep) -> Add(Mod, Dep); - (F, Mod, compile, {parse_transform, Dep}) -> Add(Mod, Dep); - (F, Mod, compile, Opts) when is_list(Opts) -> - case proplists:get_value(parse_transform, Opts) of - undefined -> ok; - Dep -> Add(Mod, Dep) - end; - (F, Mod, include, Hrl) -> - case filelib:is_file("include/" ++ Hrl) of - true -> AddHd(F, Mod, "include/" ++ Hrl); - false -> - case filelib:is_file("src/" ++ Hrl) of - true -> AddHd(F, Mod, "src/" ++ Hrl); - false -> false - end - end; - (F, Mod, include_lib, "$1/include/" ++ Hrl) -> AddHd(F, Mod, "include/" ++ Hrl); - (F, Mod, include_lib, Hrl) -> AddHd(F, Mod, "include/" ++ Hrl); - (F, Mod, import, {Imp, _}) -> - case filelib:is_file("src/" ++ atom_to_list(Imp) ++ ".erl") of - false -> ok; - true -> Add(Mod, Imp) - end; - (_, _, _, _) -> ok - end, - MakeDepend = fun(F, Fd, Mod) -> - case io:parse_erl_form(Fd, undefined) of - {ok, {attribute, _, Key, Value}, _} -> - Attr(F, Mod, Key, Value), - F(F, Fd, Mod); - {eof, _} -> - file:close(Fd); - _ -> - F(F, Fd, Mod) - end - end, - [begin - Mod = list_to_atom(filename:basename(F, ".erl")), - {ok, Fd} = file:open(F, [read]), - MakeDepend(MakeDepend, Fd, Mod) - end || F <- ErlFiles], - Depend = sofs:to_external(sofs:relation_to_family(sofs:relation(ets:tab2list(E)))), - CompileFirst = [X || X <- lists:reverse(digraph_utils:topsort(G)), [] =/= digraph:in_neighbours(G, X)], - ok = file:write_file("$(1)", [ - [[F, "::", [[" ", D] || D <- Deps], "; @touch \$$@\n"] || {F, Deps} <- Depend], - "\nCOMPILE_FIRST +=", [[" ", atom_to_list(CF)] || CF <- CompileFirst], "\n" - ]), - halt() -endef - -ifeq ($(if $(NO_MAKEDEP),$(wildcard $(PROJECT).d),),) -$(PROJECT).d:: $(ERL_FILES) $(call core_find,include/,*.hrl) - $(makedep_verbose) $(call erlang,$(call makedep.erl,$@)) -endif - -# Rebuild everything when the Makefile changes. -$(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES):: $(MAKEFILE_LIST) - @touch $@ - --include $(PROJECT).d - -ebin/$(PROJECT).app:: ebin/ - -ebin/: - $(verbose) mkdir -p ebin/ - -define compile_erl - $(erlc_verbose) erlc -v $(if $(IS_DEP),$(filter-out -Werror,$(ERLC_OPTS)),$(ERLC_OPTS)) -o ebin/ \ - -pa ebin/ -I include/ $(filter-out $(ERLC_EXCLUDE_PATHS),$(COMPILE_FIRST_PATHS) $(1)) -endef - -ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES) $(wildcard src/$(PROJECT).app.src) - $(eval FILES_TO_COMPILE := $(filter-out src/$(PROJECT).app.src,$?)) - $(if $(strip $(FILES_TO_COMPILE)),$(call compile_erl,$(FILES_TO_COMPILE))) - $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null || true)) - $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename \ - $(filter-out $(ERLC_EXCLUDE_PATHS),$(ERL_FILES) $(CORE_FILES) $(BEAM_FILES))))))) -ifeq ($(wildcard src/$(PROJECT).app.src),) - $(app_verbose) printf "$(subst $(newline),\n,$(subst ",\",$(call app_file,$(GITDESCRIBE),$(MODULES))))" \ - > ebin/$(PROJECT).app -else - $(verbose) if [ -z "$$(grep -e '^[^%]*{\s*modules\s*,' src/$(PROJECT).app.src)" ]; then \ - echo "Empty modules entry not found in $(PROJECT).app.src. Please consult the erlang.mk README for instructions." >&2; \ - exit 1; \ - fi - $(appsrc_verbose) cat src/$(PROJECT).app.src \ - | sed "s/{[[:space:]]*modules[[:space:]]*,[[:space:]]*\[\]}/{modules, \[$(call comma_list,$(MODULES))\]}/" \ - | sed "s/{id,[[:space:]]*\"git\"}/{id, \"$(subst /,\/,$(GITDESCRIBE))\"}/" \ - > ebin/$(PROJECT).app -endif - -clean:: clean-app - -clean-app: - $(gen_verbose) rm -rf $(PROJECT).d ebin/ priv/mibs/ $(XRL_ERL_FILES) $(YRL_ERL_FILES) \ - $(addprefix include/,$(patsubst %.mib,%.hrl,$(notdir $(MIB_FILES)))) \ - $(addprefix include/,$(patsubst %.asn1,%.hrl,$(notdir $(ASN1_FILES)))) \ - $(addprefix include/,$(patsubst %.asn1,%.asn1db,$(notdir $(ASN1_FILES)))) \ - $(addprefix src/,$(patsubst %.asn1,%.erl,$(notdir $(ASN1_FILES)))) - -endif - -# Copyright (c) 2015, Viktor Söderqvist -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: docs-deps - -# Configuration. - -ALL_DOC_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(DOC_DEPS)) - -# Targets. - -$(foreach dep,$(DOC_DEPS),$(eval $(call dep_target,$(dep)))) - -ifneq ($(SKIP_DEPS),) -doc-deps: -else -doc-deps: $(ALL_DOC_DEPS_DIRS) - $(verbose) for dep in $(ALL_DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep; done -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: rel-deps - -# Configuration. - -ALL_REL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(REL_DEPS)) - -# Targets. - -$(foreach dep,$(REL_DEPS),$(eval $(call dep_target,$(dep)))) - -ifneq ($(SKIP_DEPS),) -rel-deps: -else -rel-deps: $(ALL_REL_DEPS_DIRS) - $(verbose) for dep in $(ALL_REL_DEPS_DIRS) ; do $(MAKE) -C $$dep; done -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: test-deps test-dir test-build clean-test-dir - -# Configuration. - -TEST_DIR ?= $(CURDIR)/test - -ALL_TEST_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(TEST_DEPS)) - -TEST_ERLC_OPTS ?= +debug_info +warn_export_vars +warn_shadow_vars +warn_obsolete_guard -TEST_ERLC_OPTS += -DTEST=1 - -# Targets. - -$(foreach dep,$(TEST_DEPS),$(eval $(call dep_target,$(dep)))) - -ifneq ($(SKIP_DEPS),) -test-deps: -else -test-deps: $(ALL_TEST_DEPS_DIRS) - $(verbose) for dep in $(ALL_TEST_DEPS_DIRS) ; do $(MAKE) -C $$dep IS_DEP=1; done -endif - -ifneq ($(wildcard $(TEST_DIR)),) -test-dir: - $(gen_verbose) erlc -v $(TEST_ERLC_OPTS) -I include/ -o $(TEST_DIR) \ - $(call core_find,$(TEST_DIR)/,*.erl) -pa ebin/ -endif - -ifeq ($(wildcard src),) -test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) -test-build:: clean deps test-deps - $(verbose) $(MAKE) --no-print-directory test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" -else -ifeq ($(wildcard ebin/test),) -test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) -test-build:: clean deps test-deps $(PROJECT).d - $(verbose) $(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" - $(gen_verbose) touch ebin/test -else -test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) -test-build:: deps test-deps $(PROJECT).d - $(verbose) $(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" -endif - -clean:: clean-test-dir - -clean-test-dir: -ifneq ($(wildcard $(TEST_DIR)/*.beam),) - $(gen_verbose) rm -f $(TEST_DIR)/*.beam -endif -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: rebar.config - -# We strip out -Werror because we don't want to fail due to -# warnings when used as a dependency. - -compat_prepare_erlc_opts = $(shell echo "$1" | sed 's/, */,/g') - -define compat_convert_erlc_opts -$(if $(filter-out -Werror,$1),\ - $(if $(findstring +,$1),\ - $(shell echo $1 | cut -b 2-))) -endef - -define compat_erlc_opts_to_list -[$(call comma_list,$(foreach o,$(call compat_prepare_erlc_opts,$1),$(call compat_convert_erlc_opts,$o)))] -endef - -define compat_rebar_config -{deps, [ -$(call comma_list,$(foreach d,$(DEPS),\ - $(if $(filter hex,$(call dep_fetch,$d)),\ - {$(call dep_name,$d)$(comma)"$(call dep_repo,$d)"},\ - {$(call dep_name,$d)$(comma)".*"$(comma){git,"$(call dep_repo,$d)"$(comma)"$(call dep_commit,$d)"}}))) -]}. -{erl_opts, $(call compat_erlc_opts_to_list,$(ERLC_OPTS))}. -endef - -$(eval _compat_rebar_config = $$(compat_rebar_config)) -$(eval export _compat_rebar_config) - -rebar.config: - $(gen_verbose) echo "$${_compat_rebar_config}" > rebar.config - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: asciidoc asciidoc-guide asciidoc-manual install-asciidoc distclean-asciidoc - -MAN_INSTALL_PATH ?= /usr/local/share/man -MAN_SECTIONS ?= 3 7 - -docs:: asciidoc - -asciidoc: asciidoc-guide asciidoc-manual - -ifeq ($(wildcard doc/src/guide/book.asciidoc),) -asciidoc-guide: -else -asciidoc-guide: distclean-asciidoc doc-deps - a2x -v -f pdf doc/src/guide/book.asciidoc && mv doc/src/guide/book.pdf doc/guide.pdf - a2x -v -f chunked doc/src/guide/book.asciidoc && mv doc/src/guide/book.chunked/ doc/html/ -endif - -ifeq ($(wildcard doc/src/manual/*.asciidoc),) -asciidoc-manual: -else -asciidoc-manual: distclean-asciidoc doc-deps - for f in doc/src/manual/*.asciidoc ; do \ - a2x -v -f manpage $$f ; \ - done - for s in $(MAN_SECTIONS); do \ - mkdir -p doc/man$$s/ ; \ - mv doc/src/manual/*.$$s doc/man$$s/ ; \ - gzip doc/man$$s/*.$$s ; \ - done - -install-docs:: install-asciidoc - -install-asciidoc: asciidoc-manual - for s in $(MAN_SECTIONS); do \ - mkdir -p $(MAN_INSTALL_PATH)/man$$s/ ; \ - install -g `id -u` -o `id -g` -m 0644 doc/man$$s/*.gz $(MAN_INSTALL_PATH)/man$$s/ ; \ - done -endif - -distclean:: distclean-asciidoc - -distclean-asciidoc: - $(gen_verbose) rm -rf doc/html/ doc/guide.pdf doc/man3/ doc/man7/ - -# Copyright (c) 2014-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: bootstrap bootstrap-lib bootstrap-rel new list-templates - -# Core targets. - -help:: - $(verbose) printf "%s\n" "" \ - "Bootstrap targets:" \ - " bootstrap Generate a skeleton of an OTP application" \ - " bootstrap-lib Generate a skeleton of an OTP library" \ - " bootstrap-rel Generate the files needed to build a release" \ - " new-app in=NAME Create a new local OTP application NAME" \ - " new-lib in=NAME Create a new local OTP library NAME" \ - " new t=TPL n=NAME Generate a module NAME based on the template TPL" \ - " new t=T n=N in=APP Generate a module NAME based on the template TPL in APP" \ - " list-templates List available templates" - -# Bootstrap templates. - -define bs_appsrc -{application, $p, [ - {description, ""}, - {vsn, "0.1.0"}, - {id, "git"}, - {modules, []}, - {registered, []}, - {applications, [ - kernel, - stdlib - ]}, - {mod, {$p_app, []}}, - {env, []} -]}. -endef - -define bs_appsrc_lib -{application, $p, [ - {description, ""}, - {vsn, "0.1.0"}, - {id, "git"}, - {modules, []}, - {registered, []}, - {applications, [ - kernel, - stdlib - ]} -]}. -endef - -# To prevent autocompletion issues with ZSH, we add "include erlang.mk" -# separately during the actual bootstrap. -ifdef SP -define bs_Makefile -PROJECT = $p -PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 - -# Whitespace to be used when creating files from templates. -SP = $(SP) - -endef -else -define bs_Makefile -PROJECT = $p -PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 - -endef -endif - -define bs_apps_Makefile -PROJECT = $p -PROJECT_DESCRIPTION = New project -PROJECT_VERSION = 0.0.1 - -include $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(APPS_DIR)/app)/erlang.mk -endef - -define bs_app --module($p_app). --behaviour(application). - --export([start/2]). --export([stop/1]). - -start(_Type, _Args) -> - $p_sup:start_link(). - -stop(_State) -> - ok. -endef - -define bs_relx_config -{release, {$p_release, "1"}, [$p]}. -{extended_start_script, true}. -{sys_config, "rel/sys.config"}. -{vm_args, "rel/vm.args"}. -endef - -define bs_sys_config -[ -]. -endef - -define bs_vm_args --name $p@127.0.0.1 --setcookie $p --heart -endef - -# Normal templates. - -define tpl_supervisor --module($(n)). --behaviour(supervisor). - --export([start_link/0]). --export([init/1]). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - Procs = [], - {ok, {{one_for_one, 1, 5}, Procs}}. -endef - -define tpl_gen_server --module($(n)). --behaviour(gen_server). - -%% API. --export([start_link/0]). - -%% gen_server. --export([init/1]). --export([handle_call/3]). --export([handle_cast/2]). --export([handle_info/2]). --export([terminate/2]). --export([code_change/3]). - --record(state, { -}). - -%% API. - --spec start_link() -> {ok, pid()}. -start_link() -> - gen_server:start_link(?MODULE, [], []). - -%% gen_server. - -init([]) -> - {ok, #state{}}. - -handle_call(_Request, _From, State) -> - {reply, ignored, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. -endef - -define tpl_module --module($(n)). --export([]). -endef - -define tpl_cowboy_http --module($(n)). --behaviour(cowboy_http_handler). - --export([init/3]). --export([handle/2]). --export([terminate/3]). - --record(state, { -}). - -init(_, Req, _Opts) -> - {ok, Req, #state{}}. - -handle(Req, State=#state{}) -> - {ok, Req2} = cowboy_req:reply(200, Req), - {ok, Req2, State}. - -terminate(_Reason, _Req, _State) -> - ok. -endef - -define tpl_gen_fsm --module($(n)). --behaviour(gen_fsm). - -%% API. --export([start_link/0]). - -%% gen_fsm. --export([init/1]). --export([state_name/2]). --export([handle_event/3]). --export([state_name/3]). --export([handle_sync_event/4]). --export([handle_info/3]). --export([terminate/3]). --export([code_change/4]). - --record(state, { -}). - -%% API. - --spec start_link() -> {ok, pid()}. -start_link() -> - gen_fsm:start_link(?MODULE, [], []). - -%% gen_fsm. - -init([]) -> - {ok, state_name, #state{}}. - -state_name(_Event, StateData) -> - {next_state, state_name, StateData}. - -handle_event(_Event, StateName, StateData) -> - {next_state, StateName, StateData}. - -state_name(_Event, _From, StateData) -> - {reply, ignored, state_name, StateData}. - -handle_sync_event(_Event, _From, StateName, StateData) -> - {reply, ignored, StateName, StateData}. - -handle_info(_Info, StateName, StateData) -> - {next_state, StateName, StateData}. - -terminate(_Reason, _StateName, _StateData) -> - ok. - -code_change(_OldVsn, StateName, StateData, _Extra) -> - {ok, StateName, StateData}. -endef - -define tpl_cowboy_loop --module($(n)). --behaviour(cowboy_loop_handler). - --export([init/3]). --export([info/3]). --export([terminate/3]). - --record(state, { -}). - -init(_, Req, _Opts) -> - {loop, Req, #state{}, 5000, hibernate}. - -info(_Info, Req, State) -> - {loop, Req, State, hibernate}. - -terminate(_Reason, _Req, _State) -> - ok. -endef - -define tpl_cowboy_rest --module($(n)). - --export([init/3]). --export([content_types_provided/2]). --export([get_html/2]). - -init(_, _Req, _Opts) -> - {upgrade, protocol, cowboy_rest}. - -content_types_provided(Req, State) -> - {[{{<<"text">>, <<"html">>, '*'}, get_html}], Req, State}. - -get_html(Req, State) -> - {<<"This is REST!">>, Req, State}. -endef - -define tpl_cowboy_ws --module($(n)). --behaviour(cowboy_websocket_handler). - --export([init/3]). --export([websocket_init/3]). --export([websocket_handle/3]). --export([websocket_info/3]). --export([websocket_terminate/3]). - --record(state, { -}). - -init(_, _, _) -> - {upgrade, protocol, cowboy_websocket}. - -websocket_init(_, Req, _Opts) -> - Req2 = cowboy_req:compact(Req), - {ok, Req2, #state{}}. - -websocket_handle({text, Data}, Req, State) -> - {reply, {text, Data}, Req, State}; -websocket_handle({binary, Data}, Req, State) -> - {reply, {binary, Data}, Req, State}; -websocket_handle(_Frame, Req, State) -> - {ok, Req, State}. - -websocket_info(_Info, Req, State) -> - {ok, Req, State}. - -websocket_terminate(_Reason, _Req, _State) -> - ok. -endef - -define tpl_ranch_protocol --module($(n)). --behaviour(ranch_protocol). - --export([start_link/4]). --export([init/4]). - --type opts() :: []. --export_type([opts/0]). - --record(state, { - socket :: inet:socket(), - transport :: module() -}). - -start_link(Ref, Socket, Transport, Opts) -> - Pid = spawn_link(?MODULE, init, [Ref, Socket, Transport, Opts]), - {ok, Pid}. - --spec init(ranch:ref(), inet:socket(), module(), opts()) -> ok. -init(Ref, Socket, Transport, _Opts) -> - ok = ranch:accept_ack(Ref), - loop(#state{socket=Socket, transport=Transport}). - -loop(State) -> - loop(State). -endef - -# Plugin-specific targets. - -define render_template - $(verbose) printf -- '$(subst $(newline),\n,$(subst %,%%,$(subst ','\'',$(subst $(tab),$(WS),$(call $(1))))))\n' > $(2) -endef - -ifndef WS -ifdef SP -WS = $(subst a,,a $(wordlist 1,$(SP),a a a a a a a a a a a a a a a a a a a a)) -else -WS = $(tab) -endif -endif - -bootstrap: -ifneq ($(wildcard src/),) - $(error Error: src/ directory already exists) -endif - $(eval p := $(PROJECT)) - $(eval n := $(PROJECT)_sup) - $(call render_template,bs_Makefile,Makefile) - $(verbose) echo "include erlang.mk" >> Makefile - $(verbose) mkdir src/ -ifdef LEGACY - $(call render_template,bs_appsrc,src/$(PROJECT).app.src) -endif - $(call render_template,bs_app,src/$(PROJECT)_app.erl) - $(call render_template,tpl_supervisor,src/$(PROJECT)_sup.erl) - -bootstrap-lib: -ifneq ($(wildcard src/),) - $(error Error: src/ directory already exists) -endif - $(eval p := $(PROJECT)) - $(call render_template,bs_Makefile,Makefile) - $(verbose) echo "include erlang.mk" >> Makefile - $(verbose) mkdir src/ -ifdef LEGACY - $(call render_template,bs_appsrc_lib,src/$(PROJECT).app.src) -endif - -bootstrap-rel: -ifneq ($(wildcard relx.config),) - $(error Error: relx.config already exists) -endif -ifneq ($(wildcard rel/),) - $(error Error: rel/ directory already exists) -endif - $(eval p := $(PROJECT)) - $(call render_template,bs_relx_config,relx.config) - $(verbose) mkdir rel/ - $(call render_template,bs_sys_config,rel/sys.config) - $(call render_template,bs_vm_args,rel/vm.args) - -new-app: -ifndef in - $(error Usage: $(MAKE) new-app in=APP) -endif -ifneq ($(wildcard $(APPS_DIR)/$in),) - $(error Error: Application $in already exists) -endif - $(eval p := $(in)) - $(eval n := $(in)_sup) - $(verbose) mkdir -p $(APPS_DIR)/$p/src/ - $(call render_template,bs_apps_Makefile,$(APPS_DIR)/$p/Makefile) -ifdef LEGACY - $(call render_template,bs_appsrc,$(APPS_DIR)/$p/src/$p.app.src) -endif - $(call render_template,bs_app,$(APPS_DIR)/$p/src/$p_app.erl) - $(call render_template,tpl_supervisor,$(APPS_DIR)/$p/src/$p_sup.erl) - -new-lib: -ifndef in - $(error Usage: $(MAKE) new-lib in=APP) -endif -ifneq ($(wildcard $(APPS_DIR)/$in),) - $(error Error: Application $in already exists) -endif - $(eval p := $(in)) - $(verbose) mkdir -p $(APPS_DIR)/$p/src/ - $(call render_template,bs_apps_Makefile,$(APPS_DIR)/$p/Makefile) -ifdef LEGACY - $(call render_template,bs_appsrc_lib,$(APPS_DIR)/$p/src/$p.app.src) -endif - -new: -ifeq ($(wildcard src/)$(in),) - $(error Error: src/ directory does not exist) -endif -ifndef t - $(error Usage: $(MAKE) new t=TEMPLATE n=NAME [in=APP]) -endif -ifndef tpl_$(t) - $(error Unknown template) -endif -ifndef n - $(error Usage: $(MAKE) new t=TEMPLATE n=NAME [in=APP]) -endif -ifdef in - $(verbose) $(MAKE) -C $(APPS_DIR)/$(in)/ new t=$t n=$n in= -else - $(call render_template,tpl_$(t),src/$(n).erl) -endif - -list-templates: - $(verbose) echo Available templates: $(sort $(patsubst tpl_%,%,$(filter tpl_%,$(.VARIABLES)))) - -# Copyright (c) 2014-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: clean-c_src distclean-c_src-env - -# Configuration. - -C_SRC_DIR ?= $(CURDIR)/c_src -C_SRC_ENV ?= $(C_SRC_DIR)/env.mk -C_SRC_OUTPUT ?= $(CURDIR)/priv/$(PROJECT) -C_SRC_TYPE ?= shared - -# System type and C compiler/flags. - -ifeq ($(PLATFORM),msys2) - C_SRC_OUTPUT_EXECUTABLE_EXTENSION ?= .exe - C_SRC_OUTPUT_SHARED_EXTENSION ?= .dll -else - C_SRC_OUTPUT_EXECUTABLE_EXTENSION ?= - C_SRC_OUTPUT_SHARED_EXTENSION ?= .so -endif - -ifeq ($(C_SRC_TYPE),shared) - C_SRC_OUTPUT_FILE = $(C_SRC_OUTPUT)$(C_SRC_OUTPUT_SHARED_EXTENSION) -else - C_SRC_OUTPUT_FILE = $(C_SRC_OUTPUT)$(C_SRC_OUTPUT_EXECUTABLE_EXTENSION) -endif - -ifeq ($(PLATFORM),msys2) -# We hardcode the compiler used on MSYS2. The default CC=cc does -# not produce working code. The "gcc" MSYS2 package also doesn't. - CC = /mingw64/bin/gcc - export CC - CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes - CXXFLAGS ?= -O3 -finline-functions -Wall -else ifeq ($(PLATFORM),darwin) - CC ?= cc - CFLAGS ?= -O3 -std=c99 -arch x86_64 -finline-functions -Wall -Wmissing-prototypes - CXXFLAGS ?= -O3 -arch x86_64 -finline-functions -Wall - LDFLAGS ?= -arch x86_64 -flat_namespace -undefined suppress -else ifeq ($(PLATFORM),freebsd) - CC ?= cc - CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes - CXXFLAGS ?= -O3 -finline-functions -Wall -else ifeq ($(PLATFORM),linux) - CC ?= gcc - CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes - CXXFLAGS ?= -O3 -finline-functions -Wall -endif - -ifneq ($(PLATFORM),msys2) - CFLAGS += -fPIC - CXXFLAGS += -fPIC -endif - -CFLAGS += -I"$(ERTS_INCLUDE_DIR)" -I"$(ERL_INTERFACE_INCLUDE_DIR)" -CXXFLAGS += -I"$(ERTS_INCLUDE_DIR)" -I"$(ERL_INTERFACE_INCLUDE_DIR)" - -LDLIBS += -L"$(ERL_INTERFACE_LIB_DIR)" -lerl_interface -lei - -# Verbosity. - -c_verbose_0 = @echo " C " $(?F); -c_verbose = $(c_verbose_$(V)) - -cpp_verbose_0 = @echo " CPP " $(?F); -cpp_verbose = $(cpp_verbose_$(V)) - -link_verbose_0 = @echo " LD " $(@F); -link_verbose = $(link_verbose_$(V)) - -# Targets. - -ifeq ($(wildcard $(C_SRC_DIR)),) -else ifneq ($(wildcard $(C_SRC_DIR)/Makefile),) -app:: app-c_src - -test-build:: app-c_src - -app-c_src: - $(MAKE) -C $(C_SRC_DIR) - -clean:: - $(MAKE) -C $(C_SRC_DIR) clean - -else - -ifeq ($(SOURCES),) -SOURCES := $(sort $(foreach pat,*.c *.C *.cc *.cpp,$(call core_find,$(C_SRC_DIR)/,$(pat)))) -endif -OBJECTS = $(addsuffix .o, $(basename $(SOURCES))) - -COMPILE_C = $(c_verbose) $(CC) $(CFLAGS) $(CPPFLAGS) -c -COMPILE_CPP = $(cpp_verbose) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c - -app:: $(C_SRC_ENV) $(C_SRC_OUTPUT_FILE) - -test-build:: $(C_SRC_ENV) $(C_SRC_OUTPUT_FILE) - -$(C_SRC_OUTPUT_FILE): $(OBJECTS) - $(verbose) mkdir -p priv/ - $(link_verbose) $(CC) $(OBJECTS) \ - $(LDFLAGS) $(if $(filter $(C_SRC_TYPE),shared),-shared) $(LDLIBS) \ - -o $(C_SRC_OUTPUT_FILE) - -%.o: %.c - $(COMPILE_C) $(OUTPUT_OPTION) $< - -%.o: %.cc - $(COMPILE_CPP) $(OUTPUT_OPTION) $< - -%.o: %.C - $(COMPILE_CPP) $(OUTPUT_OPTION) $< - -%.o: %.cpp - $(COMPILE_CPP) $(OUTPUT_OPTION) $< - -clean:: clean-c_src - -clean-c_src: - $(gen_verbose) rm -f $(C_SRC_OUTPUT_FILE) $(OBJECTS) - -endif - -ifneq ($(wildcard $(C_SRC_DIR)),) -$(C_SRC_ENV): - $(verbose) $(ERL) -eval "file:write_file(\"$(call core_native_path,$(C_SRC_ENV))\", \ - io_lib:format( \ - \"ERTS_INCLUDE_DIR ?= ~s/erts-~s/include/~n\" \ - \"ERL_INTERFACE_INCLUDE_DIR ?= ~s~n\" \ - \"ERL_INTERFACE_LIB_DIR ?= ~s~n\", \ - [code:root_dir(), erlang:system_info(version), \ - code:lib_dir(erl_interface, include), \ - code:lib_dir(erl_interface, lib)])), \ - halt()." - -distclean:: distclean-c_src-env - -distclean-c_src-env: - $(gen_verbose) rm -f $(C_SRC_ENV) - --include $(C_SRC_ENV) -endif - -# Templates. - -define bs_c_nif -#include "erl_nif.h" - -static int loads = 0; - -static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - /* Initialize private data. */ - *priv_data = NULL; - - loads++; - - return 0; -} - -static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) -{ - /* Convert the private data to the new version. */ - *priv_data = *old_priv_data; - - loads++; - - return 0; -} - -static void unload(ErlNifEnv* env, void* priv_data) -{ - if (loads == 1) { - /* Destroy the private data. */ - } - - loads--; -} - -static ERL_NIF_TERM hello(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - if (enif_is_atom(env, argv[0])) { - return enif_make_tuple2(env, - enif_make_atom(env, "hello"), - argv[0]); - } - - return enif_make_tuple2(env, - enif_make_atom(env, "error"), - enif_make_atom(env, "badarg")); -} - -static ErlNifFunc nif_funcs[] = { - {"hello", 1, hello} -}; - -ERL_NIF_INIT($n, nif_funcs, load, NULL, upgrade, unload) -endef - -define bs_erl_nif --module($n). - --export([hello/1]). - --on_load(on_load/0). -on_load() -> - PrivDir = case code:priv_dir(?MODULE) of - {error, _} -> - AppPath = filename:dirname(filename:dirname(code:which(?MODULE))), - filename:join(AppPath, "priv"); - Path -> - Path - end, - erlang:load_nif(filename:join(PrivDir, atom_to_list(?MODULE)), 0). - -hello(_) -> - erlang:nif_error({not_loaded, ?MODULE}). -endef - -new-nif: -ifneq ($(wildcard $(C_SRC_DIR)/$n.c),) - $(error Error: $(C_SRC_DIR)/$n.c already exists) -endif -ifneq ($(wildcard src/$n.erl),) - $(error Error: src/$n.erl already exists) -endif -ifdef in - $(verbose) $(MAKE) -C $(APPS_DIR)/$(in)/ new-nif n=$n in= -else - $(verbose) mkdir -p $(C_SRC_DIR) src/ - $(call render_template,bs_c_nif,$(C_SRC_DIR)/$n.c) - $(call render_template,bs_erl_nif,src/$n.erl) -endif - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: ci ci-setup distclean-kerl - -KERL ?= $(CURDIR)/kerl -export KERL - -KERL_URL ?= https://raw.githubusercontent.com/yrashk/kerl/master/kerl - -OTP_GIT ?= https://github.com/erlang/otp - -CI_INSTALL_DIR ?= $(HOME)/erlang -CI_OTP ?= - -ifeq ($(strip $(CI_OTP)),) -ci:: -else -ci:: $(addprefix ci-,$(CI_OTP)) - -ci-prepare: $(addprefix $(CI_INSTALL_DIR)/,$(CI_OTP)) - -ci-setup:: - -ci_verbose_0 = @echo " CI " $(1); -ci_verbose = $(ci_verbose_$(V)) - -define ci_target -ci-$(1): $(CI_INSTALL_DIR)/$(1) - $(ci_verbose) \ - PATH="$(CI_INSTALL_DIR)/$(1)/bin:$(PATH)" \ - CI_OTP_RELEASE="$(1)" \ - CT_OPTS="-label $(1)" \ - $(MAKE) clean ci-setup tests -endef - -$(foreach otp,$(CI_OTP),$(eval $(call ci_target,$(otp)))) - -define ci_otp_target -ifeq ($(wildcard $(CI_INSTALL_DIR)/$(1)),) -$(CI_INSTALL_DIR)/$(1): $(KERL) - $(KERL) build git $(OTP_GIT) $(1) $(1) - $(KERL) install $(1) $(CI_INSTALL_DIR)/$(1) -endif -endef - -$(foreach otp,$(CI_OTP),$(eval $(call ci_otp_target,$(otp)))) - -$(KERL): - $(gen_verbose) $(call core_http_get,$(KERL),$(KERL_URL)) - $(verbose) chmod +x $(KERL) - -help:: - $(verbose) printf "%s\n" "" \ - "Continuous Integration targets:" \ - " ci Run '$(MAKE) tests' on all configured Erlang versions." \ - "" \ - "The CI_OTP variable must be defined with the Erlang versions" \ - "that must be tested. For example: CI_OTP = OTP-17.3.4 OTP-17.5.3" - -distclean:: distclean-kerl - -distclean-kerl: - $(gen_verbose) rm -rf $(KERL) -endif - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: ct apps-ct distclean-ct - -# Configuration. - -CT_OPTS ?= -ifneq ($(wildcard $(TEST_DIR)),) - CT_SUITES ?= $(sort $(subst _SUITE.erl,,$(notdir $(call core_find,$(TEST_DIR)/,*_SUITE.erl)))) -else - CT_SUITES ?= -endif - -# Core targets. - -tests:: ct - -distclean:: distclean-ct - -help:: - $(verbose) printf "%s\n" "" \ - "Common_test targets:" \ - " ct Run all the common_test suites for this project" \ - "" \ - "All your common_test suites have their associated targets." \ - "A suite named http_SUITE can be ran using the ct-http target." - -# Plugin-specific targets. - -CT_RUN = ct_run \ - -no_auto_compile \ - -noinput \ - -pa $(CURDIR)/ebin $(DEPS_DIR)/*/ebin $(APPS_DIR)/*/ebin $(TEST_DIR) \ - -dir $(TEST_DIR) \ - -logdir $(CURDIR)/logs - -ifeq ($(CT_SUITES),) -ct: $(if $(IS_APP),,apps-ct) -else -ct: test-build $(if $(IS_APP),,apps-ct) - $(verbose) mkdir -p $(CURDIR)/logs/ - $(gen_verbose) $(CT_RUN) -sname ct_$(PROJECT) -suite $(addsuffix _SUITE,$(CT_SUITES)) $(CT_OPTS) -endif - -ifneq ($(ALL_APPS_DIRS),) -define ct_app_target -apps-ct-$1: - $(MAKE) -C $1 ct IS_APP=1 -endef - -$(foreach app,$(ALL_APPS_DIRS),$(eval $(call ct_app_target,$(app)))) - -apps-ct: test-build $(addprefix apps-ct-,$(ALL_APPS_DIRS)) -endif - -ifndef t -CT_EXTRA = -else -ifeq (,$(findstring :,$t)) -CT_EXTRA = -group $t -else -t_words = $(subst :, ,$t) -CT_EXTRA = -group $(firstword $(t_words)) -case $(lastword $(t_words)) -endif -endif - -define ct_suite_target -ct-$(1): test-build - $(verbose) mkdir -p $(CURDIR)/logs/ - $(gen_verbose) $(CT_RUN) -sname ct_$(PROJECT) -suite $(addsuffix _SUITE,$(1)) $(CT_EXTRA) $(CT_OPTS) -endef - -$(foreach test,$(CT_SUITES),$(eval $(call ct_suite_target,$(test)))) - -distclean-ct: - $(gen_verbose) rm -rf $(CURDIR)/logs/ - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: plt distclean-plt dialyze - -# Configuration. - -DIALYZER_PLT ?= $(CURDIR)/.$(PROJECT).plt -export DIALYZER_PLT - -PLT_APPS ?= -DIALYZER_DIRS ?= --src -r $(wildcard src) $(ALL_APPS_DIRS) -DIALYZER_OPTS ?= -Werror_handling -Wrace_conditions -Wunmatched_returns # -Wunderspecs - -# Core targets. - -check:: dialyze - -distclean:: distclean-plt - -help:: - $(verbose) printf "%s\n" "" \ - "Dialyzer targets:" \ - " plt Build a PLT file for this project" \ - " dialyze Analyze the project using Dialyzer" - -# Plugin-specific targets. - -define filter_opts.erl - Opts = binary:split(<<"$1">>, <<"-">>, [global]), - Filtered = lists:reverse(lists:foldl(fun - (O = <<"pa ", _/bits>>, Acc) -> [O|Acc]; - (O = <<"D ", _/bits>>, Acc) -> [O|Acc]; - (O = <<"I ", _/bits>>, Acc) -> [O|Acc]; - (_, Acc) -> Acc - end, [], Opts)), - io:format("~s~n", [[["-", O] || O <- Filtered]]), - halt(). -endef - -$(DIALYZER_PLT): deps app - $(verbose) dialyzer --build_plt --apps erts kernel stdlib $(PLT_APPS) $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS) - -plt: $(DIALYZER_PLT) - -distclean-plt: - $(gen_verbose) rm -f $(DIALYZER_PLT) - -ifneq ($(wildcard $(DIALYZER_PLT)),) -dialyze: -else -dialyze: $(DIALYZER_PLT) -endif - $(verbose) dialyzer --no_native `$(call erlang,$(call filter_opts.erl,$(ERLC_OPTS)))` $(DIALYZER_DIRS) $(DIALYZER_OPTS) - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: distclean-edoc edoc - -# Configuration. - -EDOC_OPTS ?= - -# Core targets. - -ifneq ($(wildcard doc/overview.edoc),) -docs:: edoc -endif - -distclean:: distclean-edoc - -# Plugin-specific targets. - -edoc: distclean-edoc doc-deps - $(gen_verbose) $(ERL) -eval 'edoc:application($(PROJECT), ".", [$(EDOC_OPTS)]), halt().' - -distclean-edoc: - $(gen_verbose) rm -f doc/*.css doc/*.html doc/*.png doc/edoc-info - -# Copyright (c) 2014 Dave Cottlehuber -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: distclean-escript escript - -# Configuration. - -ESCRIPT_NAME ?= $(PROJECT) -ESCRIPT_FILE ?= $(ESCRIPT_NAME) - -ESCRIPT_COMMENT ?= This is an -*- erlang -*- file - -ESCRIPT_BEAMS ?= "ebin/*", "deps/*/ebin/*" -ESCRIPT_SYS_CONFIG ?= "rel/sys.config" -ESCRIPT_EMU_ARGS ?= -pa . \ - -sasl errlog_type error \ - -escript main $(ESCRIPT_NAME) -ESCRIPT_SHEBANG ?= /usr/bin/env escript -ESCRIPT_STATIC ?= "deps/*/priv/**", "priv/**" - -# Core targets. - -distclean:: distclean-escript - -help:: - $(verbose) printf "%s\n" "" \ - "Escript targets:" \ - " escript Build an executable escript archive" \ - -# Plugin-specific targets. - -# Based on https://github.com/synrc/mad/blob/master/src/mad_bundle.erl -# Copyright (c) 2013 Maxim Sokhatsky, Synrc Research Center -# Modified MIT License, https://github.com/synrc/mad/blob/master/LICENSE : -# Software may only be used for the great good and the true happiness of all -# sentient beings. - -define ESCRIPT_RAW -'Read = fun(F) -> {ok, B} = file:read_file(filename:absname(F)), B end,'\ -'Files = fun(L) -> A = lists:concat([filelib:wildcard(X)||X<- L ]),'\ -' [F || F <- A, not filelib:is_dir(F) ] end,'\ -'Squash = fun(L) -> [{filename:basename(F), Read(F) } || F <- L ] end,'\ -'Zip = fun(A, L) -> {ok,{_,Z}} = zip:create(A, L, [{compress,all},memory]), Z end,'\ -'Ez = fun(Escript) ->'\ -' Static = Files([$(ESCRIPT_STATIC)]),'\ -' Beams = Squash(Files([$(ESCRIPT_BEAMS), $(ESCRIPT_SYS_CONFIG)])),'\ -' Archive = Beams ++ [{ "static.gz", Zip("static.gz", Static)}],'\ -' escript:create(Escript, [ $(ESCRIPT_OPTIONS)'\ -' {archive, Archive, [memory]},'\ -' {shebang, "$(ESCRIPT_SHEBANG)"},'\ -' {comment, "$(ESCRIPT_COMMENT)"},'\ -' {emu_args, " $(ESCRIPT_EMU_ARGS)"}'\ -' ]),'\ -' file:change_mode(Escript, 8#755)'\ -'end,'\ -'Ez("$(ESCRIPT_FILE)"),'\ -'halt().' -endef - -ESCRIPT_COMMAND = $(subst ' ',,$(ESCRIPT_RAW)) - -escript:: distclean-escript deps app - $(gen_verbose) $(ERL) -eval $(ESCRIPT_COMMAND) - -distclean-escript: - $(gen_verbose) rm -f $(ESCRIPT_NAME) - -# Copyright (c) 2014, Enrique Fernandez -# Copyright (c) 2015, Loïc Hoguin -# This file is contributed to erlang.mk and subject to the terms of the ISC License. - -.PHONY: eunit apps-eunit - -# Configuration - -EUNIT_OPTS ?= -EUNIT_ERL_OPTS ?= - -# Core targets. - -tests:: eunit - -help:: - $(verbose) printf "%s\n" "" \ - "EUnit targets:" \ - " eunit Run all the EUnit tests for this project" - -# Plugin-specific targets. - -define eunit.erl - case "$(COVER)" of - "" -> ok; - _ -> - case cover:compile_beam_directory("ebin") of - {error, _} -> halt(1); - _ -> ok - end - end, - case eunit:test($1, [$(EUNIT_OPTS)]) of - ok -> ok; - error -> halt(2) - end, - case "$(COVER)" of - "" -> ok; - _ -> - cover:export("eunit.coverdata") - end, - halt() -endef - -EUNIT_ERL_OPTS += -pa $(TEST_DIR) $(DEPS_DIR)/*/ebin $(APPS_DIR)/*/ebin $(CURDIR)/ebin - -ifdef t -ifeq (,$(findstring :,$(t))) -eunit: test-build - $(gen_verbose) $(call erlang,$(call eunit.erl,['$(t)']),$(EUNIT_ERL_OPTS)) -else -eunit: test-build - $(gen_verbose) $(call erlang,$(call eunit.erl,fun $(t)/0),$(EUNIT_ERL_OPTS)) -endif -else -EUNIT_EBIN_MODS = $(notdir $(basename $(ERL_FILES) $(BEAM_FILES))) -EUNIT_TEST_MODS = $(notdir $(basename $(call core_find,$(TEST_DIR)/,*.erl))) - -EUNIT_MODS = $(foreach mod,$(EUNIT_EBIN_MODS) $(filter-out \ - $(patsubst %,%_tests,$(EUNIT_EBIN_MODS)),$(EUNIT_TEST_MODS)),'$(mod)') - -eunit: test-build $(if $(IS_APP),,apps-eunit) - $(gen_verbose) $(call erlang,$(call eunit.erl,[$(call comma_list,$(EUNIT_MODS))]),$(EUNIT_ERL_OPTS)) - -ifneq ($(ALL_APPS_DIRS),) -apps-eunit: - $(verbose) for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app eunit IS_APP=1; done -endif -endif - -# Copyright (c) 2013-2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: relx-rel distclean-relx-rel distclean-relx run - -# Configuration. - -RELX ?= $(CURDIR)/relx -RELX_CONFIG ?= $(CURDIR)/relx.config - -RELX_URL ?= https://github.com/erlware/relx/releases/download/v3.19.0/relx -RELX_OPTS ?= -RELX_OUTPUT_DIR ?= _rel - -ifeq ($(firstword $(RELX_OPTS)),-o) - RELX_OUTPUT_DIR = $(word 2,$(RELX_OPTS)) -else - RELX_OPTS += -o $(RELX_OUTPUT_DIR) -endif - -# Core targets. - -ifeq ($(IS_DEP),) -ifneq ($(wildcard $(RELX_CONFIG)),) -rel:: relx-rel -endif -endif - -distclean:: distclean-relx-rel distclean-relx - -# Plugin-specific targets. - -$(RELX): - $(gen_verbose) $(call core_http_get,$(RELX),$(RELX_URL)) - $(verbose) chmod +x $(RELX) - -relx-rel: $(RELX) rel-deps app - $(verbose) $(RELX) -c $(RELX_CONFIG) $(RELX_OPTS) - -distclean-relx-rel: - $(gen_verbose) rm -rf $(RELX_OUTPUT_DIR) - -distclean-relx: - $(gen_verbose) rm -rf $(RELX) - -# Run target. - -ifeq ($(wildcard $(RELX_CONFIG)),) -run: -else - -define get_relx_release.erl - {ok, Config} = file:consult("$(RELX_CONFIG)"), - {release, {Name, _}, _} = lists:keyfind(release, 1, Config), - io:format("~s", [Name]), - halt(0). -endef - -RELX_RELEASE = `$(call erlang,$(get_relx_release.erl))` - -run: all - $(verbose) $(RELX_OUTPUT_DIR)/$(RELX_RELEASE)/bin/$(RELX_RELEASE) console - -help:: - $(verbose) printf "%s\n" "" \ - "Relx targets:" \ - " run Compile the project, build the release and run it" - -endif - -# Copyright (c) 2014, M Robert Martin -# Copyright (c) 2015, Loïc Hoguin -# This file is contributed to erlang.mk and subject to the terms of the ISC License. - -.PHONY: shell - -# Configuration. - -SHELL_ERL ?= erl -SHELL_PATHS ?= $(CURDIR)/ebin $(APPS_DIR)/*/ebin $(DEPS_DIR)/*/ebin -SHELL_OPTS ?= - -ALL_SHELL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(SHELL_DEPS)) - -# Core targets - -help:: - $(verbose) printf "%s\n" "" \ - "Shell targets:" \ - " shell Run an erlang shell with SHELL_OPTS or reasonable default" - -# Plugin-specific targets. - -$(foreach dep,$(SHELL_DEPS),$(eval $(call dep_target,$(dep)))) - -build-shell-deps: $(ALL_SHELL_DEPS_DIRS) - $(verbose) for dep in $(ALL_SHELL_DEPS_DIRS) ; do $(MAKE) -C $$dep ; done - -shell: build-shell-deps - $(gen_verbose) $(SHELL_ERL) -pa $(SHELL_PATHS) $(SHELL_OPTS) - -# Copyright (c) 2015, Loïc Hoguin -# This file is part of erlang.mk and subject to the terms of the ISC License. - -ifeq ($(filter triq,$(DEPS) $(TEST_DEPS)),triq) -.PHONY: triq - -# Targets. - -tests:: triq - -define triq_check.erl - code:add_pathsa(["$(CURDIR)/ebin", "$(DEPS_DIR)/*/ebin"]), - try - case $(1) of - all -> [true] =:= lists:usort([triq:check(M) || M <- [$(call comma_list,$(3))]]); - module -> triq:check($(2)); - function -> triq:check($(2)) - end - of - true -> halt(0); - _ -> halt(1) - catch error:undef -> - io:format("Undefined property or module~n"), - halt(0) - end. -endef - -ifdef t -ifeq (,$(findstring :,$(t))) -triq: test-build - $(verbose) $(call erlang,$(call triq_check.erl,module,$(t))) -else -triq: test-build - $(verbose) echo Testing $(t)/0 - $(verbose) $(call erlang,$(call triq_check.erl,function,$(t)())) -endif -else -triq: test-build - $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename $(wildcard ebin/*.beam)))))) - $(gen_verbose) $(call erlang,$(call triq_check.erl,all,undefined,$(MODULES))) -endif -endif - -# Copyright (c) 2015, Erlang Solutions Ltd. -# This file is part of erlang.mk and subject to the terms of the ISC License. - -.PHONY: xref distclean-xref - -# Configuration. - -ifeq ($(XREF_CONFIG),) - XREF_ARGS := -else - XREF_ARGS := -c $(XREF_CONFIG) -endif - -XREFR ?= $(CURDIR)/xrefr -export XREFR - -XREFR_URL ?= https://github.com/inaka/xref_runner/releases/download/0.2.2/xrefr - -# Core targets. - -help:: - $(verbose) printf "%s\n" "" \ - "Xref targets:" \ - " xref Run Xrefr using $XREF_CONFIG as config file if defined" - -distclean:: distclean-xref - -# Plugin-specific targets. - -$(XREFR): - $(gen_verbose) $(call core_http_get,$(XREFR),$(XREFR_URL)) - $(verbose) chmod +x $(XREFR) - -xref: deps app $(XREFR) - $(gen_verbose) $(XREFR) $(XREFR_ARGS) - -distclean-xref: - $(gen_verbose) rm -rf $(XREFR) - -# Copyright 2015, Viktor Söderqvist -# This file is part of erlang.mk and subject to the terms of the ISC License. - -COVER_REPORT_DIR = cover - -# Hook in coverage to ct - -ifdef COVER -ifdef CT_RUN -# All modules in 'ebin' -COVER_MODS = $(notdir $(basename $(call core_ls,ebin/*.beam))) - -test-build:: $(TEST_DIR)/ct.cover.spec - -$(TEST_DIR)/ct.cover.spec: - $(verbose) echo Cover mods: $(COVER_MODS) - $(gen_verbose) printf "%s\n" \ - '{incl_mods,[$(subst $(space),$(comma),$(COVER_MODS))]}.' \ - '{export,"$(CURDIR)/ct.coverdata"}.' > $@ - -CT_RUN += -cover $(TEST_DIR)/ct.cover.spec -endif -endif - -# Core targets - -ifdef COVER -ifneq ($(COVER_REPORT_DIR),) -tests:: - $(verbose) $(MAKE) --no-print-directory cover-report -endif -endif - -clean:: coverdata-clean - -ifneq ($(COVER_REPORT_DIR),) -distclean:: cover-report-clean -endif - -help:: - $(verbose) printf "%s\n" "" \ - "Cover targets:" \ - " cover-report Generate a HTML coverage report from previously collected" \ - " cover data." \ - " all.coverdata Merge {eunit,ct}.coverdata into one coverdata file." \ - "" \ - "If COVER=1 is set, coverage data is generated by the targets eunit and ct. The" \ - "target tests additionally generates a HTML coverage report from the combined" \ - "coverdata files from each of these testing tools. HTML reports can be disabled" \ - "by setting COVER_REPORT_DIR to empty." - -# Plugin specific targets - -COVERDATA = $(filter-out all.coverdata,$(wildcard *.coverdata)) - -.PHONY: coverdata-clean -coverdata-clean: - $(gen_verbose) rm -f *.coverdata ct.cover.spec - -# Merge all coverdata files into one. -all.coverdata: $(COVERDATA) - $(gen_verbose) $(ERL) -eval ' \ - $(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) \ - cover:export("$@"), halt(0).' - -# These are only defined if COVER_REPORT_DIR is non-empty. Set COVER_REPORT_DIR to -# empty if you want the coverdata files but not the HTML report. -ifneq ($(COVER_REPORT_DIR),) - -.PHONY: cover-report-clean cover-report - -cover-report-clean: - $(gen_verbose) rm -rf $(COVER_REPORT_DIR) - -ifeq ($(COVERDATA),) -cover-report: -else - -# Modules which include eunit.hrl always contain one line without coverage -# because eunit defines test/0 which is never called. We compensate for this. -EUNIT_HRL_MODS = $(subst $(space),$(comma),$(shell \ - grep -e '^\s*-include.*include/eunit\.hrl"' src/*.erl \ - | sed "s/^src\/\(.*\)\.erl:.*/'\1'/" | uniq)) - -define cover_report.erl - $(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) - Ms = cover:imported_modules(), - [cover:analyse_to_file(M, "$(COVER_REPORT_DIR)/" ++ atom_to_list(M) - ++ ".COVER.html", [html]) || M <- Ms], - Report = [begin {ok, R} = cover:analyse(M, module), R end || M <- Ms], - EunitHrlMods = [$(EUNIT_HRL_MODS)], - Report1 = [{M, {Y, case lists:member(M, EunitHrlMods) of - true -> N - 1; false -> N end}} || {M, {Y, N}} <- Report], - TotalY = lists:sum([Y || {_, {Y, _}} <- Report1]), - TotalN = lists:sum([N || {_, {_, N}} <- Report1]), - Perc = fun(Y, N) -> case Y + N of 0 -> 100; S -> round(100 * Y / S) end end, - TotalPerc = Perc(TotalY, TotalN), - {ok, F} = file:open("$(COVER_REPORT_DIR)/index.html", [write]), - io:format(F, "~n" - "~n" - "Coverage report~n" - "~n", []), - io:format(F, "

Coverage

~n

Total: ~p%

~n", [TotalPerc]), - io:format(F, "~n", []), - [io:format(F, "" - "~n", - [M, M, Perc(Y, N)]) || {M, {Y, N}} <- Report1], - How = "$(subst $(space),$(comma)$(space),$(basename $(COVERDATA)))", - Date = "$(shell date -u "+%Y-%m-%dT%H:%M:%SZ")", - io:format(F, "
ModuleCoverage
~p~p%
~n" - "

Generated using ~s and erlang.mk on ~s.

~n" - "", [How, Date]), - halt(). -endef - -cover-report: - $(gen_verbose) mkdir -p $(COVER_REPORT_DIR) - $(gen_verbose) $(call erlang,$(cover_report.erl)) - -endif -endif # ifneq ($(COVER_REPORT_DIR),) diff --git a/etc/acl.conf b/etc/acl.conf index 2560bf80d..b60ea5e7a 100644 --- a/etc/acl.conf +++ b/etc/acl.conf @@ -1,6 +1,6 @@ %%-------------------------------------------------------------------- %% -%% [ACL](https://github.com/emqtt/emqttd/wiki/ACL) +%% [ACL](http://emqtt.io/docs/v2/config.html#allow-anonymous-and-acl-file) %% %% -type who() :: all | binary() | %% {ipaddr, esockd_access:cidr()} | @@ -24,3 +24,4 @@ {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. +{allow, all}. diff --git a/etc/acl.conf.paho b/etc/acl.conf.paho new file mode 100644 index 000000000..5beec4347 --- /dev/null +++ b/etc/acl.conf.paho @@ -0,0 +1,14 @@ +%%-------------------------------------------------------------------- +%% For paho interoperability test cases +%%-------------------------------------------------------------------- + +{deny, {client, "myclientid"}, subscribe, ["test/nosubscribe"]}. + +{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. + +{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. + +{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. + +{allow, all}. + diff --git a/etc/emq.conf b/etc/emqx.conf similarity index 52% rename from etc/emq.conf rename to etc/emqx.conf index f296b8ff9..cc319e5f5 100644 --- a/etc/emq.conf +++ b/etc/emqx.conf @@ -1,5 +1,5 @@ ##==================================================================== -## EMQ Configuration R2 +## EMQ X Configuration R3.0 ##==================================================================== ##-------------------------------------------------------------------- @@ -7,7 +7,9 @@ ##-------------------------------------------------------------------- ## Cluster name. -cluster.name = emqcl +## +## Value: String +cluster.name = emqxcl ## Cluster auto-discovery strategy. ## @@ -46,7 +48,7 @@ cluster.autoclean = 5m ## Node list of the cluster. ## ## Value: String -## cluster.static.seeds = emq1@127.0.0.1,emq2@127.0.0.1 +## cluster.static.seeds = emqx1@127.0.0.1,emqx2@127.0.0.1 ##-------------------------------------------------------------------- ## Cluster using IP Multicast. @@ -89,7 +91,7 @@ cluster.autoclean = 5m ## The App name is used to build 'node.name' with IP address. ## ## Value: String -## cluster.dns.app = emq +## cluster.dns.app = emqx ##-------------------------------------------------------------------- ## Cluster using etcd @@ -103,7 +105,7 @@ cluster.autoclean = 5m ## will create a path in etcd: v2/keys/// ## ## Value: String -## cluster.etcd.prefix = emqcl +## cluster.etcd.prefix = emqxcl ## The TTL for node's path in etcd. ## @@ -123,7 +125,7 @@ cluster.autoclean = 5m ## The service name helps lookup EMQ nodes in the cluster. ## ## Value: String -## cluster.k8s.service_name = emq +## cluster.k8s.service_name = emqx ## The address type is used to extract host from k8s service. ## @@ -133,7 +135,12 @@ cluster.autoclean = 5m ## The app name helps build 'node.name'. ## ## Value: String -## cluster.k8s.app_name = emq +## cluster.k8s.app_name = emqx + +## Kubernates Namespace +## +## Value: String +## cluster.k8s.namespace = default ## Kubernates Namespace ## @@ -142,7 +149,7 @@ cluster.autoclean = 5m ##-------------------------------------------------------------------- -## Node Args +## Node ##-------------------------------------------------------------------- ## Node name. @@ -151,18 +158,13 @@ cluster.autoclean = 5m ## ## Value: @ ## -## Default: emq@127.0.0.1 -node.name = emq@127.0.0.1 +## Default: emqx@127.0.0.1 +node.name = emqx@127.0.0.1 ## Cookie for distributed node communication. ## ## Value: String -node.cookie = emqsecretcookie - -## Enable SMP support of Erlang VM. -## -## Value: enable | auto | disable -node.smp = auto +node.cookie = emqxsecretcookie ## Heartbeat monitoring of an Erlang runtime system. Comment the line to disable ## heartbeat, or set the value as 'on' @@ -172,13 +174,6 @@ node.smp = auto ## vm.args: -heart ## node.heartbeat = on -## Enable kernel poll. -## -## Value: on | off -## -## Default: on -node.kernel_poll = on - ## Sets the number of threads in async thread pool. Valid range is 0-1024. ## ## See: http://erlang.org/doc/man/erl.html @@ -196,17 +191,16 @@ node.async_threads = 32 ## Value: Number [1024-134217727] ## ## vm.args: +P Number -node.process_limit = 256000 +node.process_limit = 2048000 -## Sets the maximum number of simultaneously existing ports for this system -## if a Number is passed as value. +## Sets the maximum number of simultaneously existing ports for this system. ## ## See: http://erlang.org/doc/man/erl.html ## ## Value: Number [1024-134217727] ## ## vm.args: +Q Number -node.max_ports = 65536 +node.max_ports = 1024000 ## Set the distribution buffer busy limit (dist_buf_busy_limit). ## @@ -273,220 +267,346 @@ node.dist_net_ticktime = 60 ## ## Value: Port [1024-65535] node.dist_listen_min = 6369 -node.dist_listen_max = 6379 +node.dist_listen_max = 6369 + +##-------------------------------------------------------------------- +## RPC +##-------------------------------------------------------------------- + +## TCP server port for RPC. +## +## Value: Port [1024-65535] +rpc.tcp_server_port = 5369 + +## TCP port for outgoing RPC connections. +## +## Value: Port [1024-65535] +rpc.tcp_client_port = 5369 + +## RCP Client connect timeout. +## +## Value: Seconds +rpc.connect_timeout = 5000 + +## TCP send timeout of RPC client and server. +## +## Value: Seconds +rpc.send_timeout = 5000 + +## Authentication timeout +## +## Value: Seconds +rpc.authentication_timeout = 5000 + +## Default receive timeout for call() functions +## +## Value: Seconds +rpc.call_receive_timeout = 15000 + +## Socket idle keepalive. +## +## Value: Seconds +rpc.socket_keepalive_idle = 900 + +## TCP Keepalive probes interval. +## +## Value: Integer +rpc.socket_keepalive_interval = 75 + +## Probes lost to close the connection +## +## Value: Integer +rpc.socket_keepalive_count = 9 ##-------------------------------------------------------------------- ## Log ##-------------------------------------------------------------------- -## Sets the log dir. +## Where to emit the logs. +## Enable the console (standard output) logs. +## +## Value: off | file | console | both +## - off: disable logs entirely +## - file: write logs only to file +## - console: write logs only to standard I/O +## - both: write logs both to file and standard I/O +log.to = both + +## The log severity level. +## +## Value: debug | info | notice | warning | error | critical | alert | emergency +## +## Note: Only the messages with severity level higher than or equal to +## this level will be logged. +## +## Default: error +log.level = error + +## The dir for log files. ## ## Value: Folder log.dir = {{ platform_log_dir }} -## Where to emit the console logs. +## The log filename for logs of level specified in "log.level". ## -## Value: off | file | console | both -## - off: disabled -## - file: write to file -## - console: write to stdout -## - both: file and stdout -log.console = console +## Value: String +## Default: emqx.log +log.file = emqx.log -## Sets the severity level of console log. -## -## Value: debug | info | notice | warning | error | critical | alert | emergency -## -## Default: error -log.console.level = error - -## The file where console logs will be writed to, when 'log.console' is set as 'file'. -## -## Value: File Name -## log.console.file = {{ platform_log_dir }}/console.log - -## Maximum file size for console log. -## -## Value: Number(bytes) -## log.console.size = 10485760 - -## The rotation count for console log. +## Maximum size of each log file. ## ## Value: Number -## log.console.count = 5 +## Default: 10M +## Supported Unit: KB | MB | G +log.rotation.size = 10MB -## The file where info logs will be writed to. -## -## Value: File Name -## log.info.file = {{ platform_log_dir }}/info.log - -## Maximum file size for info log. -## -## Value: Number(bytes) -## log.info.size = 10485760 - -## The rotation count for info log. +## Maximum rotation count of log files. ## ## Value: Number -## log.info.count = 5 +## Default: 5 +log.rotation.count = 5 -## The file where error logs will be writed to. +## To create additional log files for specific log levels. ## ## Value: File Name -log.error.file = {{ platform_log_dir }}/error.log - -## Maximum file size for error log. +## Format: log.$level.file = $filename, +## where "$level" can be one of: debug, info, notice, warning, +## error, critical, alert, emergency +## Note: Log files for a specific log level will only contain all the logs +## that higher than or equal to that level ## -## Value: Number(bytes) -log.error.size = 10485760 - -## The rotation count for error log. -## -## Value: Number -log.error.count = 5 - -## Enable the crash log. -## -## Value: on | off -log.crash = on - -## The file for crash log. -## -## Value: File Name -log.crash.file = {{ platform_log_dir }}/crash.log - -## Enable syslog. -## -## Values: on | off -log.syslog = on - -## Sets the severity level for syslog. -## -## Value: debug | info | notice | warning | error | critical | alert | emergency -log.syslog.level = error +#log.info.file = info.log +#log.error.file = error.log ##-------------------------------------------------------------------- -## Allow Anonymous Authentication and Default ACL +## Authentication/Access Control ##-------------------------------------------------------------------- -## Allow Anonymous Authentication. -## -## Notice: Disable the option for production deployment. +## Allow anonymous authentication by default if no auth plugins loaded. +## Notice: Disable the option in production deployment! ## ## Value: true | false -mqtt.allow_anonymous = true +allow_anonymous = true -## Default behaviour when ACL nomatch. +## Allow or deny if no ACL rules matched. ## ## Value: allow | deny -mqtt.acl_nomatch = allow +acl_nomatch = allow ## Default ACL File. ## ## Value: File Name -mqtt.acl_file = {{ platform_etc_dir }}/acl.conf +acl_file = {{ platform_etc_dir }}/acl.conf -## Whether to cache ACL for publish messages. +## Whether to enable ACL cache. ## -## Value: true | false -mqtt.cache_acl = true +## If enabled, ACLs roles for each client will be cached in the memory +## +## Value: on | off +enable_acl_cache = on + +## The maximum count of ACL entries can be cached for a client. +## +## Value: Integer greater than 0 +## Default: 32 +acl_cache_max_size = 32 + +## The time after which an ACL cache entry will be deleted +## +## Value: Duration +## Default: 1 minute +acl_cache_ttl = 1m + +## The action when acl check reject current operation +## +## Value: ignore | disconnect +## Default: ignore +acl_deny_action = ignore ##-------------------------------------------------------------------- ## MQTT Protocol ##-------------------------------------------------------------------- -## Maximum length of MQTT clientId allowed. +## Response Topic Prefix ## -## Value: Number [23-65535] -mqtt.max_clientid_len = 1024 +## Value: String +## Default: emqxrspv1 +mqtt.response_topic_prefix = emqxrspv1 ## Maximum MQTT packet size allowed. ## ## Value: Bytes +## Default: 1MB +mqtt.max_packet_size = 1MB + +## Maximum length of MQTT clientId allowed. ## -## Default: 64K -mqtt.max_packet_size = 64KB +## Value: Number [23-65535] +mqtt.max_clientid_len = 65535 -## Check if the websocket protocol header is valid. -## Turn off the option when developing WeChat App. -## -## Value: on | off -mqtt.websocket_protocol_header = on - -## The backoff for MQTT keepalive timeout. -## EMQ will kick a MQTT connection out until 'Keepalive * backoff * 2' timeout. -## -## Value: Float > 0.5 -mqtt.keepalive_backoff = 0.75 - -##-------------------------------------------------------------------- -## MQTT Connection -##-------------------------------------------------------------------- - -## Force GC the MQTT connections. Value 0 will disable the Force GC. -## -## Value: Number >= 0 -mqtt.conn.force_gc_count = 100 - -##-------------------------------------------------------------------- -## MQTT Client -##-------------------------------------------------------------------- - -## MQTT client idle timeout, specified in seconds. -## -## Value: Duration -mqtt.client.idle_timeout = 30s - -## TODO: Maximum publish rate of MQTT messages per second. +## Maximum topic levels allowed. 0 means no limit. ## ## Value: Number -## mqtt.client.max_publish_rate = 5 +mqtt.max_topic_levels = 0 -## Enable per client statistics. +## Maximum QoS allowed. ## -## Value: on | off -mqtt.client.enable_stats = off +## Value: 0 | 1 | 2 +mqtt.max_qos_allowed = 2 + +## Maximum Topic Alias, 0 means no topic alias supported. +## +## Value: 0-65535 +mqtt.max_topic_alias = 0 + +## Whether the Server supports MQTT retained messages. +## +## Value: boolean +mqtt.retain_available = true + +## Whether the Server supports MQTT Wildcard Subscriptions +## +## Value: boolean +mqtt.wildcard_subscription = true + +## Whether the Server supports MQTT Shared Subscriptions. +## +## Value: boolean +mqtt.shared_subscription = true + +## Whether to ignore loop delivery of messages.(for mqtt v3.1.1) +## +## Value: true | false +mqtt.ignore_loop_deliver = false ##-------------------------------------------------------------------- -## MQTT Session +## Zones ##-------------------------------------------------------------------- +##-------------------------------------------------------------------- +## External Zone + +## Idle timeout of the external MQTT connections. +## +## Value: duration +zone.external.idle_timeout = 15s + +## Publish limit for the external MQTT connections. +## +## Value: Number,Duration +## Example: 10 messages per minute. +## zone.external.publish_limit = 10,1m + +## Enable ACL check. +## +## Value: Flag +zone.external.enable_acl = on + +## Enable ban check. +## +## Value: Flag +zone.external.enable_ban = on + +## Enable per connection statistics. +## +## Value: on | off +zone.external.enable_stats = on + +## The action when acl check reject current operation +## +## Value: ignore | disconnect +## Default: ignore +zone.external.acl_deny_action = ignore + +## Force MQTT connection/session process GC after this number of +## messages | bytes passed through. +## +## Numbers delimited by `|'. Zero or negative is to disable. +zone.external.force_gc_policy = 1000|1MB + +## Maximum MQTT packet size allowed. +## +## Value: Bytes +## Default: 1MB +## zone.external.max_packet_size = 64KB + +## Maximum length of MQTT clientId allowed. +## +## Value: Number [23-65535] +## zone.external.max_clientid_len = 1024 + +## Maximum topic levels allowed. 0 means no limit. +## +## Value: Number +## zone.external.max_topic_levels = 7 + +## Maximum QoS allowed. +## +## Value: 0 | 1 | 2 +## zone.external.max_qos_allowed = 2 + +## Maximum Topic Alias, 0 means no limit. +## +## Value: 0-65535 +## zone.external.max_topic_alias = 0 + +## Whether the Server supports retained messages. +## +## Value: boolean +## zone.external.retain_available = true + +## Whether the Server supports Wildcard Subscriptions +## +## Value: boolean +## zone.external.wildcard_subscription = false + +## Whether the Server supports Shared Subscriptions +## +## Value: boolean +## zone.external.shared_subscription = false + +## Server Keep Alive +## +## Value: Number +## zone.external.server_keepalive = 0 + +## The backoff for MQTT keepalive timeout. The broker will kick a connection out +## until 'Keepalive * backoff * 2' timeout. +## +## Value: Float > 0.5 +zone.external.keepalive_backoff = 0.75 + ## Maximum number of subscriptions allowed, 0 means no limit. ## ## Value: Number -mqtt.session.max_subscriptions = 0 +zone.external.max_subscriptions = 0 ## Force to upgrade QoS according to subscription. ## ## Value: on | off -mqtt.session.upgrade_qos = off +zone.external.upgrade_qos = off ## Maximum size of the Inflight Window storing QoS1/2 messages delivered but unacked. ## ## Value: Number -mqtt.session.max_inflight = 32 +zone.external.max_inflight = 32 ## Retry interval for QoS1/2 message delivering. ## ## Value: Duration -mqtt.session.retry_interval = 20s +zone.external.retry_interval = 20s ## Maximum QoS2 packets (Client -> Broker) awaiting PUBREL, 0 means no limit. ## ## Value: Number -mqtt.session.max_awaiting_rel = 1000 +zone.external.max_awaiting_rel = 100 ## The QoS2 messages (Client -> Broker) will be dropped if awaiting PUBREL timeout. ## ## Value: Duration -mqtt.session.await_rel_timeout = 30s +zone.external.await_rel_timeout = 300s -## Enable per session statistics. -## -## Value: on | off -mqtt.session.enable_stats = on - -## Session expiration time. +## Default session expiry interval for MQTT V3.1.1 connections. ## ## Value: Duration ## -d: day @@ -495,110 +615,131 @@ mqtt.session.enable_stats = on ## -s: second ## ## Default: 2h, 2 hours -mqtt.session.expiry_interval = 2h - -## Whether to ignore loop delivery of messages. -## -## Value: true | false -## -## Default: false -mqtt.session.ignore_loop_deliver = false - -##-------------------------------------------------------------------- -## MQTT Message Queue -##-------------------------------------------------------------------- - -## Message queue type. -## -## Value: simple | priority -mqtt.mqueue.type = simple - -## Topic priority. Default is 0. -## -## Value: Number [0-255] -## -## mqtt.mqueue.priority = topic/1=10,topic/2=8 +zone.external.session_expiry_interval = 2h ## Maximum queue length. Enqueued messages when persistent client disconnected, ## or inflight window is full. 0 means no limit. ## ## Value: Number >= 0 -mqtt.mqueue.max_length = 1000 +zone.external.max_mqueue_len = 1000 -## Low-water mark of queued messages. +## Topic priorities. +## 'none' to indicate no priority table (by default), hence all messages +## are treated equal ## -## Value: Percent -mqtt.mqueue.low_watermark = 20% +## Priority number [1-255] +## Example: topic/1=10,topic/2=8 +## NOTE: comma and equal signs are not allowed for priority topic names +## NOTE: messages for topics not in the priority table are treated as +## either highest or lowest priority depending on the configured +## value for mqueue_default_priority +## +zone.external.mqueue_priorities = none -## High-water mark of queued messages. +## Default to highest priority for topics not matching priority table ## -## Value: Percent -mqtt.mqueue.high_watermark = 60% +## Value: highest | lowest +zone.external.mqueue_default_priority = highest ## Whether to enqueue Qos0 messages. ## ## Value: false | true -mqtt.mqueue.store_qos0 = true +zone.external.mqueue_store_qos0 = true + +## All the topics will be prefixed with the mountpoint path if this option is enabled. +## +## Variables in mountpoint path: +## - %c: clientid +## - %u: username +## +## Value: String +## zone.external.mountpoint = devicebound/ + +## Whether use username replace client id +## +## Value: boolean +## Default: false +zone.external.use_username_as_clientid = false ##-------------------------------------------------------------------- -## MQTT Broker and PubSub -##-------------------------------------------------------------------- +## Internal Zone -## System interval of publishing $SYS messages. +zone.internal.allow_anonymous = true + +## Enable per connection stats. ## -## Value: Duration +## Value: Flag +zone.internal.enable_stats = on + +## Enable ACL check. ## -## Default: 1m, 1 minute -mqtt.broker.sys_interval = 1m +## Value: Flag +zone.internal.enable_acl = off -## The PubSub pool size. Default value should be same as scheduler numbers. +## The action when acl check reject current operation ## -## Value: Number > 1 -mqtt.pubsub.pool_size = 8 +## Value: ignore | disconnect +## Default: ignore +zone.internal.acl_deny_action = ignore -## TODO: Subscribe asynchronously. +## See zone.$name.wildcard_subscription. ## -## Value: true | false -mqtt.pubsub.async = true +## Value: boolean +## zone.internal.wildcard_subscription = true -##-------------------------------------------------------------------- -## MQTT Bridge -##-------------------------------------------------------------------- +## See zone.$name.shared_subscription. +## +## Value: boolean +## zone.internal.shared_subscription = true -## The pending message queue size of bridge. +## See zone.$name.max_subscriptions. +## +## Value: Integer +zone.internal.max_subscriptions = 0 + +## See zone.$name.max_inflight ## ## Value: Number -mqtt.bridge.max_queue_len = 10000 +zone.internal.max_inflight = 32 -## Ping interval of bridge node. +## See zone.$name.max_awaiting_rel ## -## Value: Duration -## -## Default: 1s, 1 second -mqtt.bridge.ping_down_interval = 1s +## Value: Number +zone.internal.max_awaiting_rel = 100 -##------------------------------------------------------------------- -## MQTT Plugins -##------------------------------------------------------------------- - -## The etc dir for plugins' config. +## See zone.$name.max_mqueue_len ## -## Value: Folder -mqtt.plugins.etc_dir ={{ platform_etc_dir }}/plugins/ +## Value: Number >= 0 +zone.internal.max_mqueue_len = 1000 -## The file to store loaded plugin names. +## Whether to enqueue Qos0 messages. ## -## Value: File -mqtt.plugins.loaded_file = {{ platform_data_dir }}/loaded_plugins +## Value: false | true +zone.internal.mqueue_store_qos0 = true + +## All the topics will be prefixed with the mountpoint path if this option is enabled. +## +## Variables in mountpoint path: +## - %c: clientid +## - %u: username +## +## Value: String +## zone.internal.mountpoint = cloudbound/ + +## Whether use username replace client id +## +## Value: boolean +## Default: false +zone.internal.use_username_as_clientid = false ##-------------------------------------------------------------------- -## MQTT Listeners +## Listeners ##-------------------------------------------------------------------- ##-------------------------------------------------------------------- ## MQTT/TCP - External TCP Listener for MQTT Protocol -## listener.tcp. is the IP address and port that the MQTT/TCP +## listener.tcp.$name is the IP address and port that the MQTT/TCP ## listener will bind. ## ## Value: IP:Port | Port @@ -609,31 +750,45 @@ listener.tcp.external = 0.0.0.0:1883 ## The acceptor pool for external MQTT/TCP listener. ## ## Value: Number -listener.tcp.external.acceptors = 16 +listener.tcp.external.acceptors = 8 ## Maximum number of concurrent MQTT/TCP connections. ## ## Value: Number -listener.tcp.external.max_clients = 102400 +listener.tcp.external.max_connections = 1024000 -## TODO: Zone of the external MQTT/TCP listener belonged to. +## Maximum external connections per second. +## +## Value: Number +listener.tcp.external.max_conn_rate = 1000 + +## Specify the {active, N} option for the external MQTT/TCP Socket. +## +## Value: Number +listener.tcp.external.active_n = 100 + +## Zone of the external MQTT/TCP listener belonged to. +## +## See: zone.$name.* ## ## Value: String -## listener.tcp.external.zone = external +listener.tcp.external.zone = external -## Mountpoint of the MQTT/TCP Listener. All the topics of this -## listener will be prefixed with the mount point if this option -## is enabled. +## Mountpoint of the MQTT/TCP Listener. All the topics will be prefixed +## with the mountpoint path if this option is enabled. +## +## Variables in mountpoint path: +## - %c: clientid +## - %u: username ## ## Value: String -## listener.tcp.external.mountpoint = external/ +## listener.tcp.external.mountpoint = devicebound/ -## Rate limit for the external MQTT/TCP connections. -## Format is 'burst,rate'. +## Rate limit for the external MQTT/TCP connections. Format is 'rate,burst'. ## -## Value: burst,rate -## Unit: KB/sec -## listener.tcp.external.rate_limit = 100,10 +## Value: rate,burst +## Unit: Bps +## listener.tcp.external.rate_limit = 1024,4096 ## The access control rules for the MQTT/TCP listener. ## @@ -644,7 +799,7 @@ listener.tcp.external.max_clients = 102400 ## Example: allow 192.168.0.0/24 listener.tcp.external.access.1 = allow all -## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed +## Enable the Proxy Protocol V1/2 if the EMQ X cluster is deployed ## behind HAProxy or Nginx. ## ## See: https://www.haproxy.com/blog/haproxy/proxy-protocol/ @@ -652,17 +807,16 @@ listener.tcp.external.access.1 = allow all ## Value: on | off ## listener.tcp.external.proxy_protocol = on -## Sets the timeout for proxy protocol. EMQ will close the TCP connection +## Sets the timeout for proxy protocol. EMQ X will close the TCP connection ## if no proxy protocol packet recevied within the timeout. ## ## Value: Duration ## listener.tcp.external.proxy_protocol_timeout = 3s ## Enable the option for X.509 certificate based authentication. -## EMQ will Use the PP2_SUBTYPE_SSL_CN field in Proxy Protocol V2 -## as MQTT username. +## EMQX will use the common name of certificate as MQTT username. ## -## Value: cn +## Value: cn | dn ## listener.tcp.external.peer_cert_as_username = cn ## The TCP backlog defines the maximum length that the queue of pending @@ -686,14 +840,14 @@ listener.tcp.external.send_timeout_close = on ## See: http://erlang.org/doc/man/inet.html ## ## Value: Bytes -## listener.tcp.external.recbuf = 4KB +## listener.tcp.external.recbuf = 2KB ## The TCP send buffer(os kernel) for MQTT connections. ## ## See: http://erlang.org/doc/man/inet.html ## ## Value: Bytes -## listener.tcp.external.sndbuf = 4KB +## listener.tcp.external.sndbuf = 2KB ## The size of the user-level software buffer used by the driver. ## Not to be confused with options sndbuf and recbuf, which correspond @@ -705,7 +859,7 @@ listener.tcp.external.send_timeout_close = on ## See: http://erlang.org/doc/man/inet.html ## ## Value: Bytes -## listener.tcp.external.buffer = 4KB +## listener.tcp.external.buffer = 2KB ## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. ## @@ -742,51 +896,62 @@ listener.tcp.internal.acceptors = 4 ## Maximum number of concurrent MQTT/TCP connections. ## ## Value: Number -listener.tcp.internal.max_clients = 102400 +listener.tcp.internal.max_connections = 1024000 -## TODO: Zone of the internal MQTT/TCP listener belonged to. +## Maximum internal connections per second. +## +## Value: Number +listener.tcp.internal.max_conn_rate = 1000 + +## Specify the {active, N} option for the internal MQTT/TCP Socket. +## +## Value: Number +listener.tcp.internal.active_n = 1000 + +## Zone of the internal MQTT/TCP listener belonged to. ## ## Value: String -## listener.tcp.internal.zone = internal +listener.tcp.internal.zone = internal ## Mountpoint of the MQTT/TCP Listener. ## -## See: listener.tcp..mountpoint +## See: listener.tcp.$name.mountpoint ## ## Value: String ## listener.tcp.internal.mountpoint = internal/ ## Rate limit for the internal MQTT/TCP connections. ## -## See: listener.tcp..rate_limit +## See: listener.tcp.$name.rate_limit ## -## Value: burst,rate -## listener.tcp.internal.rate_limit = 1000,100 +## Value: rate,burst +## Unit: Bps +## listener.tcp.internal.rate_limit = 1000000,2000000 ## The TCP backlog of internal MQTT/TCP Listener. ## -## See: listener.tcp..backlog +## See: listener.tcp.$name.backlog ## ## Value: Number >= 0 listener.tcp.internal.backlog = 512 ## The TCP send timeout for internal MQTT connections. ## -## See: listener.tcp..send_timeout +## See: listener.tcp.$name.send_timeout ## ## Value: Duration listener.tcp.internal.send_timeout = 5s ## Close the MQTT/TCP connection if send timeout. ## -## See: listener.tcp..send_timeout_close +## See: listener.tcp.$name.send_timeout_close ## ## Value: on | off listener.tcp.external.send_timeout_close = on ## The TCP receive buffer(os kernel) for internal MQTT connections. ## -## See: listener.tcp..recbuf +## See: listener.tcp.$name.recbuf ## ## Value: Bytes ## listener.tcp.internal.recbuf = 16KB @@ -800,21 +965,21 @@ listener.tcp.external.send_timeout_close = on ## The size of the user-level software buffer used by the driver. ## -## See: listener.tcp..buffer +## See: listener.tcp.$name.buffer ## ## Value: Bytes ## listener.tcp.internal.buffer = 16KB ## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. ## -## See: listener.tcp..tune_buffer +## See: listener.tcp.$name.tune_buffer ## ## Value: on | off ## listener.tcp.internal.tune_buffer = off ## The TCP_NODELAY flag for internal MQTT connections. ## -## See: listener.tcp..nodelay +## See: listener.tcp.$name.nodelay ## ## Value: true | false listener.tcp.internal.nodelay = false @@ -827,7 +992,7 @@ listener.tcp.internal.reuseaddr = true ##-------------------------------------------------------------------- ## MQTT/SSL - External SSL Listener for MQTT Protocol -## listener.ssl. is the IP address and port that the MQTT/SSL +## listener.ssl.$name is the IP address and port that the MQTT/SSL ## listener will bind. ## ## Value: IP:Port | Port @@ -843,41 +1008,52 @@ listener.ssl.external.acceptors = 16 ## Maximum number of concurrent MQTT/SSL connections. ## ## Value: Number -listener.ssl.external.max_clients = 1024 +listener.ssl.external.max_connections = 102400 -## TODO: Zone of the external MQTT/SSL listener belonged to. +## Maximum MQTT/SSL connections per second. +## +## Value: Number +listener.ssl.external.max_conn_rate = 500 + +## Specify the {active, N} option for the internal MQTT/SSL Socket. +## +## Value: Number +listener.ssl.external.active_n = 100 + +## Zone of the external MQTT/SSL listener belonged to. ## ## Value: String -## listener.ssl.external.zone = external +listener.ssl.external.zone = external ## Mountpoint of the MQTT/SSL Listener. ## ## Value: String -## listener.ssl.external.mountpoint = inbound/ +## listener.ssl.external.mountpoint = devicebound/ ## The access control rules for the MQTT/SSL listener. ## -## See: listener.tcp..access +## See: listener.tcp.$name.access ## ## Value: ACL Rule listener.ssl.external.access.1 = allow all ## Rate limit for the external MQTT/SSL connections. ## -## Value: burst,rate -## listener.ssl.external.rate_limit = 100,10 +## Value: rate,burst +## Unit: Bps +## listener.ssl.external.rate_limit = 1024,4096 ## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind ## HAProxy or Nginx. ## -## See: listener.tcp..proxy_protocol +## See: listener.tcp.$name.proxy_protocol ## ## Value: on | off ## listener.ssl.external.proxy_protocol = on ## Sets the timeout for proxy protocol. ## -## See: listener.tcp..proxy_protocol_timeout +## See: listener.tcp.$name.proxy_protocol_timeout ## ## Value: Duration ## listener.ssl.external.proxy_protocol_timeout = 3s @@ -964,7 +1140,7 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Most of it was copied from Mozilla’s Server Side TLS article ## ## Value: Ciphers -## listener.ssl.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA +listener.ssl.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA ## SSL parameter renegotiation is a feature that allows a client and a server ## to renegotiate the parameters of the SSL connection on the fly. @@ -991,64 +1167,64 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Value: on | off ## listener.ssl.external.honor_cipher_order = on -## Use the CN or DN value from the client certificate as a username. +## Use the CN, EN or CRT field from the client certificate as a username. ## Notice that 'verify' should be set as 'verify_peer'. ## -## Value: cn | dn +## Value: cn | en | crt ## listener.ssl.external.peer_cert_as_username = cn ## TCP backlog for the SSL connection. ## -## See listener.tcp..backlog +## See listener.tcp.$name.backlog ## ## Value: Number >= 0 ## listener.ssl.external.backlog = 1024 ## The TCP send timeout for the SSL connection. ## -## See listener.tcp..send_timeout +## See listener.tcp.$name.send_timeout ## ## Value: Duration ## listener.ssl.external.send_timeout = 15s ## Close the SSL connection if send timeout. ## -## See: listener.tcp..send_timeout_close +## See: listener.tcp.$name.send_timeout_close ## ## Value: on | off ## listener.ssl.external.send_timeout_close = on ## The TCP receive buffer(os kernel) for the SSL connections. ## -## See: listener.tcp..recbuf +## See: listener.tcp.$name.recbuf ## ## Value: Bytes ## listener.ssl.external.recbuf = 4KB ## The TCP send buffer(os kernel) for internal MQTT connections. ## -## See: listener.tcp..sndbuf +## See: listener.tcp.$name.sndbuf ## ## Value: Bytes ## listener.ssl.external.sndbuf = 4KB ## The size of the user-level software buffer used by the driver. ## -## See: listener.tcp..buffer +## See: listener.tcp.$name.buffer ## ## Value: Bytes ## listener.ssl.external.buffer = 4KB ## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. ## -## See: listener.tcp..tune_buffer +## See: listener.tcp.$name.tune_buffer ## ## Value: on | off ## listener.ssl.external.tune_buffer = off ## The TCP_NODELAY flag for SSL connections. ## -## See: listener.tcp..nodelay +## See: listener.tcp.$name.nodelay ## ## Value: true | false ## listener.ssl.external.nodelay = true @@ -1059,9 +1235,9 @@ listener.ssl.external.certfile = {{ platform_etc_dir }}/certs/cert.pem listener.ssl.external.reuseaddr = true ##-------------------------------------------------------------------- -## External WebSocket Listener for MQTT Protocol +## External WebSocket listener for MQTT protocol -## listener.ws. is the IP address and port that the MQTT/WebSocket +## listener.ws.$name is the IP address and port that the MQTT/WebSocket ## listener will bind. ## ## Value: IP:Port | Port @@ -1069,6 +1245,11 @@ listener.ssl.external.reuseaddr = true ## Examples: 8083, 127.0.0.1:8083, ::1:8083 listener.ws.external = 8083 +## The path of WebSocket MQTT endpoint +## +## Value: URL Path +listener.ws.external.mqtt_path = /mqtt + ## The acceptor pool for external MQTT/WebSocket listener. ## ## Value: Number @@ -1077,34 +1258,50 @@ listener.ws.external.acceptors = 4 ## Maximum number of concurrent MQTT/WebSocket connections. ## ## Value: Number -listener.ws.external.max_clients = 102400 +listener.ws.external.max_connections = 102400 -## TODO: Zone of the external MQTT/WebSocket listener belonged to. +## Maximum MQTT/WebSocket connections per second. +## +## Value: Number +listener.ws.external.max_conn_rate = 1000 + +## Rate limit for the MQTT/WebSocket connections. +## +## Value: rate,burst +## Unit: Bps +## listener.ws.external.rate_limit = 1024,4096 + +## Zone of the external MQTT/WebSocket listener belonged to. ## ## Value: String -## listener.ws.external.zone = external +listener.ws.external.zone = external ## Mountpoint of the MQTT/WebSocket Listener. ## -## See: listener.tcp..mountpoint +## See: listener.tcp.$name.mountpoint ## ## Value: String -## listener.ws.external.mountpoint = external/ +## listener.ws.external.mountpoint = devicebound/ ## The access control for the MQTT/WebSocket listener. ## -## See: listener.tcp..access +## See: listener.tcp.$name.access ## ## Value: ACL Rule listener.ws.external.access.1 = allow all -## Use X-Forwarded-For header for real source IP if the EMQ cluster is +## Verify if the protocol header is valid. Turn off for WeChat MiniApp. +## +## Value: on | off +listener.ws.external.verify_protocol_header = on + +## Use X-Forwarded-For header for real source IP if the EMQ X cluster is ## deployed behind NGINX or HAProxy. ## ## Value: String ## listener.ws.external.proxy_address_header = X-Forwarded-For -## Use X-Forwarded-Port header for real source port if the EMQ cluster is +## Use X-Forwarded-Port header for real source port if the EMQ X cluster is ## deployed behind NGINX or HAProxy. ## ## Value: String @@ -1113,83 +1310,78 @@ listener.ws.external.access.1 = allow all ## Enable the Proxy Protocol V1/2 if the EMQ cluster is deployed behind ## HAProxy or Nginx. ## -## See: listener.tcp..proxy_protocol +## See: listener.tcp.$name.proxy_protocol ## ## Value: on | off ## listener.ws.external.proxy_protocol = on ## Sets the timeout for proxy protocol. ## -## See: listener.tcp..proxy_protocol_timeout +## See: listener.tcp.$name.proxy_protocol_timeout ## ## Value: Duration ## listener.ws.external.proxy_protocol_timeout = 3s ## The TCP backlog of external MQTT/WebSocket Listener. ## -## See: listener.tcp..backlog +## See: listener.tcp.$name.backlog ## ## Value: Number >= 0 listener.ws.external.backlog = 1024 ## The TCP send timeout for external MQTT/WebSocket connections. ## -## See: listener.tcp..send_timeout +## See: listener.tcp.$name.send_timeout ## ## Value: Duration listener.ws.external.send_timeout = 15s ## Close the MQTT/WebSocket connection if send timeout. ## -## See: listener.tcp..send_timeout_close +## See: listener.tcp.$name.send_timeout_close ## ## Value: on | off listener.ws.external.send_timeout_close = on ## The TCP receive buffer(os kernel) for external MQTT/WebSocket connections. ## -## See: listener.tcp..recbuf +## See: listener.tcp.$name.recbuf ## ## Value: Bytes -## listener.ws.external.recbuf = 4KB +## listener.ws.external.recbuf = 2KB ## The TCP send buffer(os kernel) for external MQTT/WebSocket connections. ## -## See: listener.tcp..sndbuf +## See: listener.tcp.$name.sndbuf ## ## Value: Bytes -## listener.ws.external.sndbuf = 4KB +## listener.ws.external.sndbuf = 2KB ## The size of the user-level software buffer used by the driver. ## -## See: listener.tcp..buffer +## See: listener.tcp.$name.buffer ## ## Value: Bytes -## listener.ws.external.buffer = 4KB +## listener.ws.external.buffer = 2KB ## Sets the 'buffer = max(sndbuf, recbuf)' if this option is enabled. ## -## See: listener.tcp..tune_buffer +## See: listener.tcp.$name.tune_buffer ## ## Value: on | off ## listener.ws.external.tune_buffer = off ## The TCP_NODELAY flag for external MQTT/WebSocket connections. ## -## See: listener.tcp..nodelay +## See: listener.tcp.$name.nodelay ## ## Value: true | false listener.ws.external.nodelay = true -## The SO_REUSEADDR flag for MQTT/WebSocket Listener. -## -## Value: true | false -listener.ws.external.reuseaddr = true - ##-------------------------------------------------------------------- ## External WebSocket/SSL listener for MQTT Protocol -## listener.wss. is the IP address and port that the MQTT/WebSocket/SSL +## listener.wss.$name is the IP address and port that the MQTT/WebSocket/SSL ## listener will bind. ## ## Value: IP:Port | Port @@ -1197,6 +1389,11 @@ listener.ws.external.reuseaddr = true ## Examples: 8084, 127.0.0.1:8084, ::1:8084 listener.wss.external = 8084 +## The path of WebSocket MQTT endpoint +## +## Value: URL Path +listener.wss.external.mqtt_path = /mqtt + ## The acceptor pool for external MQTT/WebSocket/SSL listener. ## ## Value: Number @@ -1205,27 +1402,45 @@ listener.wss.external.acceptors = 4 ## Maximum number of concurrent MQTT/Webwocket/SSL connections. ## ## Value: Number -listener.wss.external.max_clients = 64 +listener.wss.external.max_connections = 16 -## TODO: Zone of the external MQTT/WebSocket/SSL listener belonged to. +## Maximum MQTT/WebSocket/SSL connections per second. +## +## See: listener.tcp.$name.max_conn_rate +## +## Value: Number +listener.wss.external.max_conn_rate = 1000 + +## Rate limit for the MQTT/WebSocket/SSL connections. +## +## Value: rate,burst +## Unit: Bps +## listener.wss.external.rate_limit = 1024,4096 + +## Zone of the external MQTT/WebSocket/SSL listener belonged to. ## ## Value: String -## listener.wss.external.zone = external +listener.wss.external.zone = external ## Mountpoint of the MQTT/WebSocket/SSL Listener. ## -## See: listener.tcp..mountpoint +## See: listener.tcp.$name.mountpoint ## ## Value: String -## listener.wss.external.mountpoint = inbound/ +## listener.wss.external.mountpoint = devicebound/ ## The access control rules for the MQTT/WebSocket/SSL listener. ## -## See: listener.tcp..access. +## See: listener.tcp.$name.access. ## ## Value: ACL Rule listener.wss.external.access.1 = allow all +## See: listener.ws.external.verify_protocol_header +## +## Value: on | off +listener.wss.external.verify_protocol_header = on + ## See: listener.ws.external.proxy_address_header ## ## Value: String @@ -1238,193 +1453,519 @@ listener.wss.external.access.1 = allow all ## Enable the Proxy Protocol V1/2 support. ## -## See: listener.tcp..proxy_protocol +## See: listener.tcp.$name.proxy_protocol ## ## Value: on | off ## listener.wss.external.proxy_protocol = on ## Sets the timeout for proxy protocol. ## -## See: listener.tcp..proxy_protocol_timeout +## See: listener.tcp.$name.proxy_protocol_timeout ## ## Value: Duration ## listener.wss.external.proxy_protocol_timeout = 3s ## TLS versions only to protect from POODLE attack. ## -## See: listener.ssl..tls_versions +## See: listener.ssl.$name.tls_versions ## ## Value: String, seperated by ',' ## listener.wss.external.tls_versions = tlsv1.2,tlsv1.1,tlsv1 -## TLS Handshake timeout. -## -## See: listener.ssl..handshake_timeout -## -## Value: Duration -listener.wss.external.handshake_timeout = 15s - ## Path to the file containing the user's private PEM-encoded key. ## -## See: listener.ssl..keyfile +## See: listener.ssl.$name.keyfile ## ## Value: File listener.wss.external.keyfile = {{ platform_etc_dir }}/certs/key.pem ## Path to a file containing the user certificate. ## -## See: listener.ssl..certfile +## See: listener.ssl.$name.certfile ## ## Value: File listener.wss.external.certfile = {{ platform_etc_dir }}/certs/cert.pem ## Path to the file containing PEM-encoded CA certificates. ## -## See: listener.ssl..cacert +## See: listener.ssl.$name.cacert ## ## Value: File ## listener.wss.external.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem -## See: listener.ssl..dhfile +## See: listener.ssl.$name.dhfile ## ## Value: File ## listener.ssl.external.dhfile = {{ platform_etc_dir }}/certs/dh-params.pem -## See: listener.ssl..vefify +## See: listener.ssl.$name.vefify ## ## Value: vefify_peer | verify_none ## listener.wss.external.verify = verify_peer -## See: listener.ssl..fail_if_no_peer_cert +## See: listener.ssl.$name.fail_if_no_peer_cert ## ## Value: false | true ## listener.wss.external.fail_if_no_peer_cert = true -## See: listener.ssl..ciphers +## See: listener.ssl.$name.ciphers ## ## Value: Ciphers ## listener.wss.external.ciphers = -## See: listener.ssl..secure_renegotiate +## See: listener.ssl.$name.secure_renegotiate ## ## Value: on | off ## listener.wss.external.secure_renegotiate = off -## See: listener.ssl..reuse_sessions +## See: listener.ssl.$name.reuse_sessions ## ## Value: on | off ## listener.wss.external.reuse_sessions = on -## See: listener.ssl..honor_cipher_order +## See: listener.ssl.$name.honor_cipher_order ## ## Value: on | off ## listener.wss.external.honor_cipher_order = on -## See: listener.ssl..peer_cert_as_username +## See: listener.ssl.$name.peer_cert_as_username ## -## Value: cn | dn +## Value: cn | dn | crt ## listener.wss.external.peer_cert_as_username = cn ## TCP backlog for the WebSocket/SSL connection. ## -## See listener.tcp..backlog +## See: listener.tcp.$name.backlog ## ## Value: Number >= 0 listener.wss.external.backlog = 1024 ## The TCP send timeout for the WebSocket/SSL connection. ## -## See: listener.tcp..send_timeout +## See: listener.tcp.$name.send_timeout ## ## Value: Duration listener.wss.external.send_timeout = 15s ## Close the WebSocket/SSL connection if send timeout. ## -## See: listener.tcp..send_timeout_close +## See: listener.tcp.$name.send_timeout_close ## ## Value: on | off listener.wss.external.send_timeout_close = on ## The TCP receive buffer(os kernel) for the WebSocket/SSL connections. ## -## See: listener.tcp..recbuf +## See: listener.tcp.$name.recbuf ## ## Value: Bytes ## listener.wss.external.recbuf = 4KB ## The TCP send buffer(os kernel) for the WebSocket/SSL connections. ## -## See: listener.tcp..sndbuf +## See: listener.tcp.$name.sndbuf ## ## Value: Bytes ## listener.wss.external.sndbuf = 4KB ## The size of the user-level software buffer used by the driver. ## -## See: listener.tcp..buffer +## See: listener.tcp.$name.buffer ## ## Value: Bytes ## listener.wss.external.buffer = 4KB ## The TCP_NODELAY flag for WebSocket/SSL connections. ## -## See: listener.tcp..nodelay +## See: listener.tcp.$name.nodelay ## ## Value: true | false ## listener.wss.external.nodelay = true -## The SO_REUSEADDR flag for WebSocket/SSL listener. -## -## Value: true | false -listener.wss.external.reuseaddr = true +listener.wss.external.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA ##-------------------------------------------------------------------- -## HTTP Management API Listener +## Bridges +##-------------------------------------------------------------------- -## The IP Address and Port that the EMQ HTTP API will bind. +##-------------------------------------------------------------------- +## Bridges to aws +##-------------------------------------------------------------------- +## Start type of the bridge. ## -## Value: IP:Port | Port -## -## Default: 0.0.0.0:8080 -listener.api.mgmt = 0.0.0.0:8080 +## Value: enum +## manual +## auto +bridge.aws.start_type = manual -## The TCP Acceptor pool size. -## -## Value: Number -listener.api.mgmt.acceptors = 4 - -## Maximum concurrent HTTP clients allowed. -## -## Value: Number -listener.api.mgmt.max_clients = 64 - -## The access control rules for the listener. -## -## See: https://github.com/emqtt/esockd#allowdeny -## -## Value: ACL Rule -listener.api.mgmt.access.1 = allow all - -## The TCP backlog for HTTP API. -## -## Value: Number >= 0 -listener.api.mgmt.backlog = 512 - -## The TCP send timeout for HTTP API. +## Bridge reconnect time. ## ## Value: Duration -listener.api.mgmt.send_timeout = 15s +## Default: 30 seconds +bridge.aws.reconnect_interval = 30s -## Close the TCP connection if send timeout. +## Bridge address: node name for local bridge, host:port for remote. +## +## Value: String +## Example: emqx@127.0.0.1, 127.0.0.1:1883 +bridge.aws.address = 127.0.0.1:1883 + +## Protocol version of the bridge. +## +## Value: Enum +## - mqttv5 +## - mqttv4 +## - mqttv3 +bridge.aws.proto_ver = mqttv4 + +## The ClientId of a remote bridge. +## +## Value: String +bridge.aws.client_id = bridge_aws + +## The Clean start flag of a remote bridge. +## +## Value: boolean +## Default: true +## +## NOTE: Some IoT platforms require clean_start +## must be set to 'true' +## bridge.aws.clean_start = true + +## The username for a remote bridge. +## +## Value: String +bridge.aws.username = user + +## The password for a remote bridge. +## +## Value: String +bridge.aws.password = passwd + +## Mountpoint of the bridge. +## +## Value: String +bridge.aws.mountpoint = bridge/aws/${node}/ + +## Ping interval of a down bridge. +## +## Value: Duration +## Default: 10 seconds +bridge.aws.keepalive = 60s + +## Forward message topics +## +## Value: String +## Example: topic1/#,topic2/# +bridge.aws.forwards = topic1/#,topic2/# + +## Subscriptions of the bridge topic. +## +## Value: String +bridge.aws.subscription.1.topic = cmd/topic1 + +## Subscriptions of the bridge qos. +## +## Value: Number +bridge.aws.subscription.1.qos = 1 + +## Subscriptions of the bridge topic. +## +## Value: String +bridge.aws.subscription.2.topic = cmd/topic2 + +## Subscriptions of the bridge qos. +## +## Value: Number +bridge.aws.subscription.2.qos = 1 + +## Bridge message queue message type. +## +## Value: Enum +## Example: memory | disk +bridge.aws.mqueue_type = memory + +## The pending message queue of a bridge. +## +## Value: Number +bridge.aws.max_pending_messages = 10000 + +## Bribge to remote server via SSL. ## ## Value: on | off -listener.api.mgmt.send_timeout_close = on +bridge.aws.ssl = off + +## PEM-encoded CA certificates of the bridge. +## +## Value: File +## bridge.aws.cacertfile = {{ platform_etc_dir }}/certs/cacert.pem + +## Client SSL Certfile of the bridge. +## +## Value: File +## bridge.aws.certfile = {{ platform_etc_dir }}/certs/client-cert.pem + +## Client SSL Keyfile of the bridge. +## +## Value: File +## bridge.aws.keyfile = {{ platform_etc_dir }}/certs/client-key.pem + +## SSL Ciphers used by the bridge. +## +## Value: String +## bridge.aws.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 + +## TLS versions used by the bridge. +## +## Value: String +## bridge.aws.tls_versions = tlsv1.2,tlsv1.1,tlsv1 + +##-------------------------------------------------------------------- +## Bridges to azure +##-------------------------------------------------------------------- +## Start type of the bridge. +## +## Value: enum +## manual +## auto +## bridge.azure.start_type = manual + +## Bridge reconnect count. +## +## Value: Number +## bridge.azure.reconnect_count = 10 + +## Bridge reconnect time. +## +## Value: Duration +## Default: 30 seconds +## bridge.azure.reconnect_time = 30s + +## Bridge address: node name for local bridge, host:port for remote. +## +## Value: String +## Example: emqx@127.0.0.1, 127.0.0.1:1883 +## bridge.azure.address = 127.0.0.1:1883 + +## Protocol version of the bridge. +## +## Value: Enum +## - mqttv5 +## - mqttv4 +## - mqttv3 +## bridge.azure.proto_ver = mqttv4 + +## The ClientId of a remote bridge. +## +## Value: String +## bridge.azure.client_id = bridge_azure + +## The Clean start flag of a remote bridge. +## +## Value: boolean +## Default: true +## +## NOTE: Some IoT platforms require clean_start +## must be set to 'true' +## bridge.azure.clean_start = true + +## The username for a remote bridge. +## +## Value: String +## bridge.azure.username = user + +## The password for a remote bridge. +## +## Value: String +## bridge.azure.password = passwd + +## Mountpoint of the bridge. +## +## Value: String +## bridge.azure.mountpoint = bridge/azure/${node}/ + +## Ping interval of a down bridge. +## +## Value: Duration +## Default: 10 seconds +## bridge.azure.keepalive = 10s + +## Forward message topics +## +## Value: String +## Example: topic1/#,topic2/# +## bridge.azure.forwards = topic1/#,topic2/# + +## Subscriptions of the bridge topic. +## +## Value: String +## bridge.azure.subscription.1.topic = $share/cmd/topic1 + +## Subscriptions of the bridge qos. +## +## Value: Number +## bridge.azure.subscription.1.qos = 1 + +## Subscriptions of the bridge topic. +## +## Value: String +## bridge.azure.subscription.2.topic = $share/cmd/topic2 + +## Subscriptions of the bridge qos. +## +## Value: Number +## bridge.azure.subscription.2.qos = 1 + +## Bridge store message type. +## +## Value: Enum +## Example: memory | disk +## bridge.azure.store_type = memory + +## The pending message queue of a bridge. +## +## Value: Number +## bridge.azure.max_pending_messages = 10000 + + +## PEM-encoded CA certificates of the bridge. +## +## Value: File +## bridge.azure.cacertfile = cacert.pem + +## Client SSL Certfile of the bridge. +## +## Value: File +## bridge.azure.certfile = cert.pem + +## Client SSL Keyfile of the bridge. +## +## Value: File +## bridge.azure.keyfile = key.pem + +## SSL Ciphers used by the bridge. +## +## Value: String +## bridge.azure.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384 + +## TLS versions used by the bridge. +## +## Value: String +## bridge.azure.tls_versions = tlsv1.2,tlsv1.1,tlsv1 + +##-------------------------------------------------------------------- +## Modules +##-------------------------------------------------------------------- + +##-------------------------------------------------------------------- +## Presence Module + +## Enable Presence Module. +## +## Value: on | off +module.presence = on + +## Sets the QoS for presence MQTT message. +## +## Value: 0 | 1 | 2 +module.presence.qos = 1 + +##-------------------------------------------------------------------- +## Subscription Module + +## Enable Subscription Module. +## +## Value: on | off +module.subscription = off + +## Subscribe the Topics automatically when client connected. +## module.subscription.1.topic = $client/%c +## Qos of the subscription: 0 | 1 | 2 +## module.subscription.1.qos = 1 + +## module.subscription.2.topic = $user/%u +## module.subscription.2.qos = 1 + +##-------------------------------------------------------------------- +## Rewrite Module + +## Enable Rewrite Module. +## +## Value: on | off +module.rewrite = off + +## {rewrite, Topic, Re, Dest} +## module.rewrite.rule.1 = x/# ^x/y/(.+)$ z/y/$1 +## module.rewrite.rule.2 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2 ##------------------------------------------------------------------- -## System Monitor +## Plugins ##------------------------------------------------------------------- +## The etc dir for plugins' config. +## +## Value: Folder +plugins.etc_dir = {{ platform_etc_dir }}/plugins/ + +## The file to store loaded plugin names. +## +## Value: File +plugins.loaded_file = {{ platform_data_dir }}/loaded_plugins + +## File to store loaded plugin names. +plugins.expand_plugins_dir = {{ platform_plugins_dir }}/ + +##-------------------------------------------------------------------- +## Broker +##-------------------------------------------------------------------- + +## System interval of publishing $SYS messages. +## +## Value: Duration +## Default: 1m, 1 minute +broker.sys_interval = 1m + +## Enable global session registry. +## +## Value: on | off +broker.enable_session_registry = on + +## Session locking strategy in a cluster. +## +## Value: Enum +## - local +## - one +## - quorum +## - all +broker.session_locking_strategy = quorum + +## Dispatch strategy for shared subscription +## +## Value: Enum +## - random +## - round_robbin +## - sticky +## - hash +broker.shared_subscription_strategy = random + +## Enable/disable shared dispatch acknowledgement for QoS1 and QoS2 messages +## This should allow messages to be dispatched to a different subscriber in +## the group in case the picked (based on shared_subscription_strategy) one # is offline +## +## Value: Enum +## - true +## - false +broker.shared_dispatch_ack_enabled = false + +## Enable batch clean for deleted routes. +## +## Value: Flag +broker.route_batch_clean = off + +##-------------------------------------------------------------------- +## System Monitor +##-------------------------------------------------------------------- + ## Enable Long GC monitoring. ## Notice: don't enable the monitor in production for: ## https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 @@ -1461,4 +2002,3 @@ sysmon.busy_port = false ## ## Value: true | false sysmon.busy_dist_port = true - diff --git a/etc/ssl_dist.conf b/etc/ssl_dist.conf index acdf0aa67..50b0e3279 100644 --- a/etc/ssl_dist.conf +++ b/etc/ssl_dist.conf @@ -1,6 +1,6 @@ %% The options in the {server, Opts} tuple are used when calling ssl:ssl_accept/3, %% and the options in the {client, Opts} tuple are used when calling ssl:connect/4. -%% +%% %% More information at: http://erlang.org/doc/apps/ssl/ssl_distribution.html [{server, [{certfile, "{{ platform_etc_dir }}/certs/cert.pem"}, diff --git a/etc/vm.args b/etc/vm.args new file mode 100644 index 000000000..43e6467d9 --- /dev/null +++ b/etc/vm.args @@ -0,0 +1,95 @@ +############################## +# Erlang VM Args +############################## + +## NOTE: +## +## Arguments configured in this file might be overridden by configs from `emqx.conf`. +## +## Some basic VM arguments are to be configured in `emqx.conf`, +## such as `node.name` for `-name` and `node.cooke` for `-setcookie`. + +## Sets the maximum number of simultaneously existing processes for this system. +#+P 2048000 + +## Sets the maximum number of simultaneously existing ports for this system. +#+Q 1024000 + +## Sets the maximum number of ETS tables +#+e 256000 + +## Sets the maximum number of atoms the virtual machine can handle. +#+t 1048576 + +## Set the location of crash dumps +#-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump + +## Set how many times generational garbages collections can be done without +## forcing a fullsweep collection. +#-env ERL_FULLSWEEP_AFTER 1000 + +## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive +## (Disabled by default..use with caution!) +#-heart + +## Specify the erlang distributed protocol. +## Can be one of: inet_tcp, inet6_tcp, inet_tls +#-proto_dist inet_tcp + +## Specify SSL Options in the file if using SSL for Erlang Distribution. +## Used only when -proto_dist set to inet_tls +#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf + +## Specifies the net_kernel tick time in seconds. +## This is the approximate time a connected node may be unresponsive until +## it is considered down and thereby disconnected. +#-kernel net_ticktime 60 + +## Sets the distribution buffer busy limit (dist_buf_busy_limit). +#+zdbbl 8192 + +## Sets default scheduler hint for port parallelism. ++spp true + +## Sets the number of threads in async thread pool. Valid range is 0-1024. +#+A 8 + +## Sets the default heap size of processes to the size Size. +#+hms 233 + +## Sets the default binary virtual heap size of processes to the size Size. +#+hmbs 46422 + +## Sets the number of IO pollsets to use when polling for I/O. +#+IOp 1 + +## Sets the number of IO poll threads to use when polling for I/O. +#+IOt 1 + +## Sets the number of scheduler threads to create and scheduler threads to set online. +#+S 8:8 + +## Sets the number of dirty CPU scheduler threads to create and dirty CPU scheduler threads to set online. +#+SDcpu 8:8 + +## Sets the number of dirty I/O scheduler threads to create. +#+SDio 10 + +## Suggested stack size, in kilowords, for scheduler threads. +#+sss 32 + +## Suggested stack size, in kilowords, for dirty CPU scheduler threads. +#+sssdcpu 40 + +## Suggested stack size, in kilowords, for dirty IO scheduler threads. +#+sssdio 40 + +## Sets scheduler bind type. +## Can be one of: u, ns, ts, ps, s, nnts, nnps, tnnps, db +#+sbt db + +## Sets a user-defined CPU topology. +#+sct L0-3c0-3p0N0:L4-7c0-3p1N1 + +## Sets the mapping of warning messages for error_logger +#+W w \ No newline at end of file diff --git a/etc/vm.args.edge b/etc/vm.args.edge new file mode 100644 index 000000000..20adc41ab --- /dev/null +++ b/etc/vm.args.edge @@ -0,0 +1,95 @@ +############################## +# Erlang VM Args +############################## + +## NOTE: +## +## Arguments configured in this file might be overridden by configs from `emqx.conf`. +## +## Some basic VM arguments are to be configured in `emqx.conf`, +## such as `node.name` for `-name` and `node.cooke` for `-setcookie`. + +## Sets the maximum number of simultaneously existing processes for this system. ++P 20480 + +## Sets the maximum number of simultaneously existing ports for this system. ++Q 4096 + +## Sets the maximum number of ETS tables ++e 512 + +## Sets the maximum number of atoms the virtual machine can handle. ++t 65536 + +## Set the location of crash dumps +-env ERL_CRASH_DUMP {{ platform_log_dir }}/crash.dump + +## Set how many times generational garbages collections can be done without +## forcing a fullsweep collection. +-env ERL_FULLSWEEP_AFTER 0 + +## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive +## (Disabled by default..use with caution!) +#-heart + +## Specify the erlang distributed protocol. +## Can be one of: inet_tcp, inet6_tcp, inet_tls +#-proto_dist inet_tcp + +## Specify SSL Options in the file if using SSL for Erlang Distribution. +## Used only when -proto_dist set to inet_tls +#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf + +## Specifies the net_kernel tick time in seconds. +## This is the approximate time a connected node may be unresponsive until +## it is considered down and thereby disconnected. +#-kernel net_ticktime 60 + +## Sets the distribution buffer busy limit (dist_buf_busy_limit). ++zdbbl 1024 + +## Sets default scheduler hint for port parallelism. ++spp false + +## Sets the number of threads in async thread pool. Valid range is 0-1024. ++A 1 + +## Sets the default heap size of processes to the size Size. +#+hms 233 + +## Sets the default binary virtual heap size of processes to the size Size. +#+hmbs 46422 + +## Sets the number of IO pollsets to use when polling for I/O. ++IOp 1 + +## Sets the number of IO poll threads to use when polling for I/O. ++IOt 1 + +## Sets the number of scheduler threads to create and scheduler threads to set online. ++S 1:1 + +## Sets the number of dirty CPU scheduler threads to create and dirty CPU scheduler threads to set online. ++SDcpu 1:1 + +## Sets the number of dirty I/O scheduler threads to create. ++SDio 1 + +## Suggested stack size, in kilowords, for scheduler threads. +#+sss 32 + +## Suggested stack size, in kilowords, for dirty CPU scheduler threads. +#+sssdcpu 40 + +## Suggested stack size, in kilowords, for dirty IO scheduler threads. +#+sssdio 40 + +## Sets scheduler bind type. +## Can be one of: u, ns, ts, ps, s, nnts, nnps, tnnps, db +#+sbt db + +## Sets a user-defined CPU topology. +#+sct L0-3c0-3p0N0:L4-7c0-3p1N1 + +## Sets the mapping of warning messages for error_logger +#+W w \ No newline at end of file diff --git a/include/emqttd.hrl b/include/emqttd.hrl deleted file mode 100644 index 975b50dd4..000000000 --- a/include/emqttd.hrl +++ /dev/null @@ -1,196 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% Banner -%%-------------------------------------------------------------------- - --define(COPYRIGHT, "Copyright (c) 2013-2018 EMQ Enterprise, Inc."). - --define(LICENSE_MESSAGE, "Licensed under the Apache License, Version 2.0"). - --define(PROTOCOL_VERSION, "MQTT/5.0"). - --define(ERTS_MINIMUM, "8.0"). - -%%-------------------------------------------------------------------- -%% Sys/Queue/Share Topics' Prefix -%%-------------------------------------------------------------------- - --define(SYSTOP, <<"$SYS/">>). %% System Topic - --define(QUEUE, <<"$queue/">>). %% Queue Topic - --define(SHARE, <<"$share/">>). %% Shared Topic - -%%-------------------------------------------------------------------- -%% PubSub -%%-------------------------------------------------------------------- - --type(pubsub() :: publish | subscribe). - --define(PS(PS), (PS =:= publish orelse PS =:= subscribe)). - -%%-------------------------------------------------------------------- -%% MQTT Topic -%%-------------------------------------------------------------------- - --record(mqtt_topic, - { topic :: binary(), - flags = [] :: [retained | static] - }). - --type(mqtt_topic() :: #mqtt_topic{}). - -%%-------------------------------------------------------------------- -%% MQTT Subscription -%%-------------------------------------------------------------------- - --record(mqtt_subscription, - { subid :: binary() | atom(), - topic :: binary(), - qos :: 0 | 1 | 2 - }). - --type(mqtt_subscription() :: #mqtt_subscription{}). - -%%-------------------------------------------------------------------- -%% MQTT Client -%%-------------------------------------------------------------------- - --type(ws_header_key() :: atom() | binary() | string()). --type(ws_header_val() :: atom() | binary() | string() | integer()). - --record(mqtt_client, - { client_id :: binary() | undefined, - client_pid :: pid(), - username :: binary() | undefined, - peername :: {inet:ip_address(), inet:port_number()}, - clean_sess :: boolean(), - proto_ver :: 3 | 4, - keepalive = 0, - will_topic :: undefined | binary(), - ws_initial_headers :: list({ws_header_key(), ws_header_val()}), - mountpoint :: undefined | binary(), - connected_at :: erlang:timestamp() - }). - --type(mqtt_client() :: #mqtt_client{}). - -%%-------------------------------------------------------------------- -%% MQTT Session -%%-------------------------------------------------------------------- - --record(mqtt_session, - { client_id :: binary(), - sess_pid :: pid(), - clean_sess :: boolean() - }). - --type(mqtt_session() :: #mqtt_session{}). - -%%-------------------------------------------------------------------- -%% MQTT Message -%%-------------------------------------------------------------------- - --type(mqtt_msg_id() :: binary() | undefined). - --type(mqtt_pktid() :: 1..16#ffff | undefined). - --type(mqtt_msg_from() :: atom() | {binary(), undefined | binary()}). - --record(mqtt_message, - { %% Global unique message ID - id :: mqtt_msg_id(), - %% PacketId - pktid :: mqtt_pktid(), - %% ClientId and Username - from :: mqtt_msg_from(), - %% Topic that the message is published to - topic :: binary(), - %% Message QoS - qos = 0 :: 0 | 1 | 2, - %% Message Flags - flags = [] :: [retain | dup | sys], - %% Retain flag - retain = false :: boolean(), - %% Dup flag - dup = false :: boolean(), - %% $SYS flag - sys = false :: boolean(), - %% Headers - headers = [] :: list(), - %% Payload - payload :: binary(), - %% Timestamp - timestamp :: erlang:timestamp() - }). - --type(mqtt_message() :: #mqtt_message{}). - -%%-------------------------------------------------------------------- -%% MQTT Delivery -%%-------------------------------------------------------------------- - --record(mqtt_delivery, - { sender :: pid(), %% Pid of the sender/publisher - message :: mqtt_message(), %% Message - flows :: list() - }). - --type(mqtt_delivery() :: #mqtt_delivery{}). - -%%-------------------------------------------------------------------- -%% MQTT Route -%%-------------------------------------------------------------------- - --record(mqtt_route, - { topic :: binary(), - node :: node() - }). - --type(mqtt_route() :: #mqtt_route{}). - -%%-------------------------------------------------------------------- -%% MQTT Alarm -%%-------------------------------------------------------------------- - --record(mqtt_alarm, - { id :: binary(), - severity :: warning | error | critical, - title :: iolist() | binary(), - summary :: iolist() | binary(), - timestamp :: erlang:timestamp() - }). - --type(mqtt_alarm() :: #mqtt_alarm{}). - -%%-------------------------------------------------------------------- -%% MQTT Plugin -%%-------------------------------------------------------------------- - --record(mqtt_plugin, { name, version, descr, active = false }). - --type(mqtt_plugin() :: #mqtt_plugin{}). - -%%-------------------------------------------------------------------- -%% MQTT CLI Command. For example: 'broker metrics' -%%-------------------------------------------------------------------- - --record(mqtt_cli, { name, action, args = [], opts = [], usage, descr }). - --type(mqtt_cli() :: #mqtt_cli{}). - diff --git a/include/emqttd_internal.hrl b/include/emqttd_internal.hrl deleted file mode 100644 index c2ae503de..000000000 --- a/include/emqttd_internal.hrl +++ /dev/null @@ -1,77 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% Internal Header File - --define(GPROC_POOL(JoinOrLeave, Pool, Id), - (begin - case JoinOrLeave of - join -> gproc_pool:connect_worker(Pool, {Pool, Id}); - leave -> gproc_pool:disconnect_worker(Pool, {Pool, Id}) - end - end)). - --define(PROC_NAME(M, I), (list_to_atom(lists:concat([M, "_", I])))). - --define(record_to_proplist(Def, Rec), - lists:zip(record_info(fields, Def), tl(tuple_to_list(Rec)))). - --define(record_to_proplist(Def, Rec, Fields), - [{K, V} || {K, V} <- ?record_to_proplist(Def, Rec), - lists:member(K, Fields)]). - --define(UNEXPECTED_REQ(Req, State), - (begin - lager:error("Unexpected Request: ~p", [Req]), - {reply, {error, unexpected_request}, State} - end)). - --define(UNEXPECTED_MSG(Msg, State), - (begin - lager:error("Unexpected Message: ~p", [Msg]), - {noreply, State} - end)). - --define(UNEXPECTED_INFO(Info, State), - (begin - lager:error("Unexpected Info: ~p", [Info]), - {noreply, State} - end)). - --define(IF(Cond, TrueFun, FalseFun), - (case (Cond) of - true -> (TrueFun); - false-> (FalseFun) - end)). - --define(FULLSWEEP_OPTS, [{fullsweep_after, 10}]). - --define(SUCCESS, 0). %% Success --define(ERROR1, 101). %% badrpc --define(ERROR2, 102). %% Unknown error --define(ERROR3, 103). %% Username or password error --define(ERROR4, 104). %% Empty username or password --define(ERROR5, 105). %% User does not exist --define(ERROR6, 106). %% Admin can not be deleted --define(ERROR7, 107). %% Missing request parameter --define(ERROR8, 108). %% Request parameter type error --define(ERROR9, 109). %% Request parameter is not a json --define(ERROR10, 110). %% Plugin has been loaded --define(ERROR11, 111). %% Plugin has been loaded --define(ERROR12, 112). %% Client not online --define(ERROR13, 113). %% User already exist --define(ERROR14, 114). %% OldPassword error - diff --git a/include/emqttd_protocol.hrl b/include/emqttd_protocol.hrl deleted file mode 100644 index 8a0ad4478..000000000 --- a/include/emqttd_protocol.hrl +++ /dev/null @@ -1,282 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% MQTT SockOpts -%%-------------------------------------------------------------------- - --define(MQTT_SOCKOPTS, [binary, {packet, raw}, {reuseaddr, true}, - {backlog, 512}, {nodelay, true}]). - -%%-------------------------------------------------------------------- -%% MQTT Protocol Version and Levels -%%-------------------------------------------------------------------- - --define(MQTT_PROTO_V3, 3). --define(MQTT_PROTO_V4, 4). --define(MQTT_PROTO_V5, 5). - --define(PROTOCOL_NAMES, [ - {?MQTT_PROTO_V3, <<"MQIsdp">>}, - {?MQTT_PROTO_V4, <<"MQTT">>}, - {?MQTT_PROTO_V5, <<"MQTT">>}]). - --type(mqtt_vsn() :: ?MQTT_PROTO_V3 | ?MQTT_PROTO_V4 | ?MQTT_PROTO_V5). - -%%-------------------------------------------------------------------- -%% MQTT QoS Level -%%-------------------------------------------------------------------- - --define(QOS_0, 0). %% At most once --define(QOS_1, 1). %% At least once --define(QOS_2, 2). %% Exactly once - --define(QOS0, 0). %% At most once --define(QOS1, 1). %% At least once --define(QOS2, 2). %% Exactly once - --define(IS_QOS(I), (I >= ?QOS0 andalso I =< ?QOS2)). - --type(mqtt_qos() :: ?QOS0 | ?QOS1 | ?QOS2). - --type(mqtt_qos_name() :: qos0 | at_most_once | - qos1 | at_least_once | - qos2 | exactly_once). - --define(QOS_I(Name), - begin - (case Name of - ?QOS_0 -> ?QOS_0; - qos0 -> ?QOS_0; - at_most_once -> ?QOS_0; - ?QOS_1 -> ?QOS_1; - qos1 -> ?QOS_1; - at_least_once -> ?QOS_1; - ?QOS_2 -> ?QOS_2; - qos2 -> ?QOS_2; - exactly_once -> ?QOS_2 - end) - end). - -%%-------------------------------------------------------------------- -%% Max ClientId Length. Why 1024? -%%-------------------------------------------------------------------- - --define(MAX_CLIENTID_LEN, 1024). - -%%-------------------------------------------------------------------- -%% MQTT Control Packet Types -%%-------------------------------------------------------------------- - --define(RESERVED, 0). %% Reserved --define(CONNECT, 1). %% Client request to connect to Server --define(CONNACK, 2). %% Server to Client: Connect acknowledgment --define(PUBLISH, 3). %% Publish message --define(PUBACK, 4). %% Publish acknowledgment --define(PUBREC, 5). %% Publish received (assured delivery part 1) --define(PUBREL, 6). %% Publish release (assured delivery part 2) --define(PUBCOMP, 7). %% Publish complete (assured delivery part 3) --define(SUBSCRIBE, 8). %% Client subscribe request --define(SUBACK, 9). %% Server Subscribe acknowledgment --define(UNSUBSCRIBE, 10). %% Unsubscribe request --define(UNSUBACK, 11). %% Unsubscribe acknowledgment --define(PINGREQ, 12). %% PING request --define(PINGRESP, 13). %% PING response --define(DISCONNECT, 14). %% Client or Server is disconnecting --define(AUTH, 15). %% Authentication exchange - --define(TYPE_NAMES, [ - 'CONNECT', - 'CONNACK', - 'PUBLISH', - 'PUBACK', - 'PUBREC', - 'PUBREL', - 'PUBCOMP', - 'SUBSCRIBE', - 'SUBACK', - 'UNSUBSCRIBE', - 'UNSUBACK', - 'PINGREQ', - 'PINGRESP', - 'DISCONNECT', - 'AUTH']). - --type(mqtt_packet_type() :: ?RESERVED..?DISCONNECT). - -%%-------------------------------------------------------------------- -%% MQTT Connect Return Codes -%%-------------------------------------------------------------------- - --define(CONNACK_ACCEPT, 0). %% Connection accepted --define(CONNACK_PROTO_VER, 1). %% Unacceptable protocol version --define(CONNACK_INVALID_ID, 2). %% Client Identifier is correct UTF-8 but not allowed by the Server --define(CONNACK_SERVER, 3). %% Server unavailable --define(CONNACK_CREDENTIALS, 4). %% Username or password is malformed --define(CONNACK_AUTH, 5). %% Client is not authorized to connect - --type(mqtt_connack() :: ?CONNACK_ACCEPT..?CONNACK_AUTH). - -%%-------------------------------------------------------------------- -%% Max MQTT Packet Length -%%-------------------------------------------------------------------- - --define(MAX_PACKET_SIZE, 16#fffffff). - -%%-------------------------------------------------------------------- -%% MQTT Parser and Serializer -%%-------------------------------------------------------------------- - --define(HIGHBIT, 2#10000000). --define(LOWBITS, 2#01111111). - -%%-------------------------------------------------------------------- -%% MQTT Packet Fixed Header -%%-------------------------------------------------------------------- - --record(mqtt_packet_header, - { type = ?RESERVED :: mqtt_packet_type(), - dup = false :: boolean(), - qos = ?QOS_0 :: mqtt_qos(), - retain = false :: boolean() - }). - -%%-------------------------------------------------------------------- -%% MQTT Packets -%%-------------------------------------------------------------------- - --type(mqtt_client_id() :: binary()). --type(mqtt_username() :: binary() | undefined). --type(mqtt_packet_id() :: 1..16#ffff | undefined). - --record(mqtt_packet_connect, - { client_id = <<>> :: mqtt_client_id(), - proto_ver = ?MQTT_PROTO_V4 :: mqtt_vsn(), - proto_name = <<"MQTT">> :: binary(), - will_retain = false :: boolean(), - will_qos = ?QOS_1 :: mqtt_qos(), - will_flag = false :: boolean(), - clean_sess = false :: boolean(), - keep_alive = 60 :: non_neg_integer(), - will_topic = undefined :: undefined | binary(), - will_msg = undefined :: undefined | binary(), - username = undefined :: undefined | binary(), - password = undefined :: undefined | binary(), - is_bridge = false :: boolean() - }). - --record(mqtt_packet_connack, - { ack_flags = ?RESERVED :: 0 | 1, - return_code :: mqtt_connack() - }). - --record(mqtt_packet_publish, - { topic_name :: binary(), - packet_id :: mqtt_packet_id() - }). - --record(mqtt_packet_puback, - { packet_id :: mqtt_packet_id() }). - --record(mqtt_packet_subscribe, - { packet_id :: mqtt_packet_id(), - topic_table :: list({binary(), mqtt_qos()}) - }). - --record(mqtt_packet_unsubscribe, - { packet_id :: mqtt_packet_id(), - topics :: list(binary()) - }). - --record(mqtt_packet_suback, - { packet_id :: mqtt_packet_id(), - qos_table :: list(mqtt_qos() | 128) - }). - --record(mqtt_packet_unsuback, - { packet_id :: mqtt_packet_id() }). - -%%-------------------------------------------------------------------- -%% MQTT Control Packet -%%-------------------------------------------------------------------- - --record(mqtt_packet, - { header :: #mqtt_packet_header{}, - variable :: #mqtt_packet_connect{} | #mqtt_packet_connack{} - | #mqtt_packet_publish{} | #mqtt_packet_puback{} - | #mqtt_packet_subscribe{} | #mqtt_packet_suback{} - | #mqtt_packet_unsubscribe{} | #mqtt_packet_unsuback{} - | mqtt_packet_id() | undefined, - payload :: binary() | undefined - }). - --type(mqtt_packet() :: #mqtt_packet{}). - -%%-------------------------------------------------------------------- -%% MQTT Packet Match -%%-------------------------------------------------------------------- - --define(CONNECT_PACKET(Var), - #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT}, variable = Var}). - --define(CONNACK_PACKET(ReturnCode), - #mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, - variable = #mqtt_packet_connack{return_code = ReturnCode}}). - --define(CONNACK_PACKET(ReturnCode, SessPresent), - #mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, - variable = #mqtt_packet_connack{ack_flags = SessPresent, - return_code = ReturnCode}}). - --define(PUBLISH_PACKET(Qos, PacketId), - #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, - qos = Qos}, - variable = #mqtt_packet_publish{packet_id = PacketId}}). - --define(PUBLISH_PACKET(Qos, Topic, PacketId, Payload), - #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, - qos = Qos}, - variable = #mqtt_packet_publish{topic_name = Topic, - packet_id = PacketId}, - payload = Payload}). - --define(PUBACK_PACKET(Type, PacketId), - #mqtt_packet{header = #mqtt_packet_header{type = Type}, - variable = #mqtt_packet_puback{packet_id = PacketId}}). - --define(PUBREL_PACKET(PacketId), - #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREL, qos = ?QOS_1}, - variable = #mqtt_packet_puback{packet_id = PacketId}}). - --define(SUBSCRIBE_PACKET(PacketId, TopicTable), - #mqtt_packet{header = #mqtt_packet_header{type = ?SUBSCRIBE, qos = ?QOS_1}, - variable = #mqtt_packet_subscribe{packet_id = PacketId, - topic_table = TopicTable}}). --define(SUBACK_PACKET(PacketId, QosTable), - #mqtt_packet{header = #mqtt_packet_header{type = ?SUBACK}, - variable = #mqtt_packet_suback{packet_id = PacketId, - qos_table = QosTable}}). --define(UNSUBSCRIBE_PACKET(PacketId, Topics), - #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBSCRIBE, qos = ?QOS_1}, - variable = #mqtt_packet_unsubscribe{packet_id = PacketId, - topics = Topics}}). --define(UNSUBACK_PACKET(PacketId), - #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBACK}, - variable = #mqtt_packet_unsuback{packet_id = PacketId}}). - --define(PACKET(Type), - #mqtt_packet{header = #mqtt_packet_header{type = Type}}). - diff --git a/include/emqttd_trie.hrl b/include/emqttd_trie.hrl deleted file mode 100644 index ffd2acebc..000000000 --- a/include/emqttd_trie.hrl +++ /dev/null @@ -1,35 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --type(trie_node_id() :: binary() | atom()). - --record(trie_node, - { node_id :: trie_node_id(), - edge_count = 0 :: non_neg_integer(), - topic :: binary() | undefined, - flags :: [retained | static] - }). - --record(trie_edge, - { node_id :: trie_node_id(), - word :: binary() | atom() - }). - --record(trie, - { edge :: #trie_edge{}, - node_id :: trie_node_id() - }). - diff --git a/include/emqx.hrl b/include/emqx.hrl new file mode 100644 index 000000000..84feb16b7 --- /dev/null +++ b/include/emqx.hrl @@ -0,0 +1,166 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-ifndef(EMQ_X_HRL). +-define(EMQ_X_HRL, true). + +%%-------------------------------------------------------------------- +%% Banner +%%-------------------------------------------------------------------- + +-define(COPYRIGHT, "Copyright (c) 2018 EMQ Technologies Co., Ltd"). + +-define(LICENSE_MESSAGE, "Licensed under the Apache License, Version 2.0"). + +-define(PROTOCOL_VERSION, "MQTT/5.0"). + +-define(ERTS_MINIMUM_REQUIRED, "10.0"). + +%%-------------------------------------------------------------------- +%% Configs +%%-------------------------------------------------------------------- + +-define(NO_PRIORITY_TABLE, none). + +%%-------------------------------------------------------------------- +%% Topics' prefix: $SYS | $queue | $share +%%-------------------------------------------------------------------- + +%% System topic +-define(SYSTOP, <<"$SYS/">>). + +%% Queue topic +-define(QUEUE, <<"$queue/">>). + +%%-------------------------------------------------------------------- +%% Message and Delivery +%%-------------------------------------------------------------------- + +-record(session, {sid, pid}). + +-record(subscription, {topic, subid, subopts}). + +%% See 'Application Message' in MQTT Version 5.0 +-record(message, { + %% Global unique message ID + id :: binary(), + %% Message QoS + qos = 0, + %% Message from + from :: atom() | binary(), + %% Message flags + flags :: #{atom() => boolean()}, + %% Message headers, or MQTT 5.0 Properties + headers = #{}, + %% Topic that the message is published to + topic :: binary(), + %% Message Payload + payload :: binary(), + %% Timestamp + timestamp :: erlang:timestamp() + }). + +-record(delivery, { + sender :: pid(), %% Sender of the delivery + message :: #message{}, %% The message delivered + results :: list() %% Dispatches of the message + }). + +%%-------------------------------------------------------------------- +%% Route +%%-------------------------------------------------------------------- + +-record(route, { + topic :: binary(), + dest :: node() | {binary(), node()} + }). + +%%-------------------------------------------------------------------- +%% Trie +%%-------------------------------------------------------------------- + +-type(trie_node_id() :: binary() | atom()). + +-record(trie_node, { + node_id :: trie_node_id(), + edge_count = 0 :: non_neg_integer(), + topic :: binary() | undefined, + flags :: list(atom()) + }). + +-record(trie_edge, { + node_id :: trie_node_id(), + word :: binary() | atom() + }). + +-record(trie, { + edge :: #trie_edge{}, + node_id :: trie_node_id() + }). + +%%-------------------------------------------------------------------- +%% Alarm +%%-------------------------------------------------------------------- + +-record(alarm, { + id :: binary(), + severity :: notice | warning | error | critical, + title :: iolist(), + summary :: iolist(), + timestamp :: erlang:timestamp() + }). + +%%-------------------------------------------------------------------- +%% Plugin +%%-------------------------------------------------------------------- + +-record(plugin, { + name :: atom(), + version :: string(), + dir :: string(), + descr :: string(), + vendor :: string(), + active = false :: boolean(), + info :: map() + }). + +%%-------------------------------------------------------------------- +%% Command +%%-------------------------------------------------------------------- + +-record(command, { + name :: atom(), + action :: atom(), + args = [] :: list(), + opts = [] :: list(), + usage :: string(), + descr :: string() + }). + +%%-------------------------------------------------------------------- +%% Banned +%%-------------------------------------------------------------------- +-type(banned_who() :: {client_id, binary()} + | {username, binary()} + | {ip_address, inet:ip_address()}). + +-record(banned, { + who :: banned_who(), + reason :: binary(), + by :: binary(), + desc :: binary(), + until :: integer() + }). + +-endif. diff --git a/include/emqx_mqtt.hrl b/include/emqx_mqtt.hrl new file mode 100644 index 000000000..3021e913a --- /dev/null +++ b/include/emqx_mqtt.hrl @@ -0,0 +1,530 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-ifndef(EMQ_X_MQTT_HRL). +-define(EMQ_X_MQTT_HRL, true). + +%%-------------------------------------------------------------------- +%% MQTT SockOpts +%%-------------------------------------------------------------------- + +-define(MQTT_SOCKOPTS, [binary, {packet, raw}, {reuseaddr, true}, + {backlog, 512}, {nodelay, true}]). + +%%-------------------------------------------------------------------- +%% MQTT Protocol Version and Names +%%-------------------------------------------------------------------- + +-define(MQTT_PROTO_V3, 3). +-define(MQTT_PROTO_V4, 4). +-define(MQTT_PROTO_V5, 5). + +-define(PROTOCOL_NAMES, [ + {?MQTT_PROTO_V3, <<"MQIsdp">>}, + {?MQTT_PROTO_V4, <<"MQTT">>}, + {?MQTT_PROTO_V5, <<"MQTT">>}]). + +%%-------------------------------------------------------------------- +%% MQTT QoS Levels +%%-------------------------------------------------------------------- + +-define(QOS_0, 0). %% At most once +-define(QOS_1, 1). %% At least once +-define(QOS_2, 2). %% Exactly once + +-define(IS_QOS(I), (I >= ?QOS_0 andalso I =< ?QOS_2)). + +-define(QOS_I(Name), + begin + (case Name of + ?QOS_0 -> ?QOS_0; + qos0 -> ?QOS_0; + at_most_once -> ?QOS_0; + ?QOS_1 -> ?QOS_1; + qos1 -> ?QOS_1; + at_least_once -> ?QOS_1; + ?QOS_2 -> ?QOS_2; + qos2 -> ?QOS_2; + exactly_once -> ?QOS_2 + end) + end). + +-define(IS_QOS_NAME(I), + (I =:= qos0 orelse I =:= at_most_once orelse + I =:= qos1 orelse I =:= at_least_once orelse + I =:= qos2 orelse I =:= exactly_once)). + +%%-------------------------------------------------------------------- +%% Maximum ClientId Length. +%%-------------------------------------------------------------------- + +-define(MAX_CLIENTID_LEN, 65535). + +%%-------------------------------------------------------------------- +%% MQTT Control Packet Types +%%-------------------------------------------------------------------- + +-define(RESERVED, 0). %% Reserved +-define(CONNECT, 1). %% Client request to connect to Server +-define(CONNACK, 2). %% Server to Client: Connect acknowledgment +-define(PUBLISH, 3). %% Publish message +-define(PUBACK, 4). %% Publish acknowledgment +-define(PUBREC, 5). %% Publish received (assured delivery part 1) +-define(PUBREL, 6). %% Publish release (assured delivery part 2) +-define(PUBCOMP, 7). %% Publish complete (assured delivery part 3) +-define(SUBSCRIBE, 8). %% Client subscribe request +-define(SUBACK, 9). %% Server Subscribe acknowledgment +-define(UNSUBSCRIBE, 10). %% Unsubscribe request +-define(UNSUBACK, 11). %% Unsubscribe acknowledgment +-define(PINGREQ, 12). %% PING request +-define(PINGRESP, 13). %% PING response +-define(DISCONNECT, 14). %% Client or Server is disconnecting +-define(AUTH, 15). %% Authentication exchange + +-define(TYPE_NAMES, [ + 'CONNECT', + 'CONNACK', + 'PUBLISH', + 'PUBACK', + 'PUBREC', + 'PUBREL', + 'PUBCOMP', + 'SUBSCRIBE', + 'SUBACK', + 'UNSUBSCRIBE', + 'UNSUBACK', + 'PINGREQ', + 'PINGRESP', + 'DISCONNECT', + 'AUTH']). + +%%-------------------------------------------------------------------- +%% MQTT V3.1.1 Connect Return Codes +%%-------------------------------------------------------------------- + +-define(CONNACK_ACCEPT, 0). %% Connection accepted +-define(CONNACK_PROTO_VER, 1). %% Unacceptable protocol version +-define(CONNACK_INVALID_ID, 2). %% Client Identifier is correct UTF-8 but not allowed by the Server +-define(CONNACK_SERVER, 3). %% Server unavailable +-define(CONNACK_CREDENTIALS, 4). %% Username or password is malformed +-define(CONNACK_AUTH, 5). %% Client is not authorized to connect + +%%-------------------------------------------------------------------- +%% MQTT V5.0 Reason Codes +%%-------------------------------------------------------------------- + +-define(RC_SUCCESS, 16#00). +-define(RC_NORMAL_DISCONNECTION, 16#00). +-define(RC_GRANTED_QOS_0, 16#00). +-define(RC_GRANTED_QOS_1, 16#01). +-define(RC_GRANTED_QOS_2, 16#02). +-define(RC_DISCONNECT_WITH_WILL_MESSAGE, 16#04). +-define(RC_NO_MATCHING_SUBSCRIBERS, 16#10). +-define(RC_NO_SUBSCRIPTION_EXISTED, 16#11). +-define(RC_CONTINUE_AUTHENTICATION, 16#18). +-define(RC_RE_AUTHENTICATE, 16#19). +-define(RC_UNSPECIFIED_ERROR, 16#80). +-define(RC_MALFORMED_PACKET, 16#81). +-define(RC_PROTOCOL_ERROR, 16#82). +-define(RC_IMPLEMENTATION_SPECIFIC_ERROR, 16#83). +-define(RC_UNSUPPORTED_PROTOCOL_VERSION, 16#84). +-define(RC_CLIENT_IDENTIFIER_NOT_VALID, 16#85). +-define(RC_BAD_USER_NAME_OR_PASSWORD, 16#86). +-define(RC_NOT_AUTHORIZED, 16#87). +-define(RC_SERVER_UNAVAILABLE, 16#88). +-define(RC_SERVER_BUSY, 16#89). +-define(RC_BANNED, 16#8A). +-define(RC_SERVER_SHUTTING_DOWN, 16#8B). +-define(RC_BAD_AUTHENTICATION_METHOD, 16#8C). +-define(RC_KEEP_ALIVE_TIMEOUT, 16#8D). +-define(RC_SESSION_TAKEN_OVER, 16#8E). +-define(RC_TOPIC_FILTER_INVALID, 16#8F). +-define(RC_TOPIC_NAME_INVALID, 16#90). +-define(RC_PACKET_IDENTIFIER_IN_USE, 16#91). +-define(RC_PACKET_IDENTIFIER_NOT_FOUND, 16#92). +-define(RC_RECEIVE_MAXIMUM_EXCEEDED, 16#93). +-define(RC_TOPIC_ALIAS_INVALID, 16#94). +-define(RC_PACKET_TOO_LARGE, 16#95). +-define(RC_MESSAGE_RATE_TOO_HIGH, 16#96). +-define(RC_QUOTA_EXCEEDED, 16#97). +-define(RC_ADMINISTRATIVE_ACTION, 16#98). +-define(RC_PAYLOAD_FORMAT_INVALID, 16#99). +-define(RC_RETAIN_NOT_SUPPORTED, 16#9A). +-define(RC_QOS_NOT_SUPPORTED, 16#9B). +-define(RC_USE_ANOTHER_SERVER, 16#9C). +-define(RC_SERVER_MOVED, 16#9D). +-define(RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED, 16#9E). +-define(RC_CONNECTION_RATE_EXCEEDED, 16#9F). +-define(RC_MAXIMUM_CONNECT_TIME, 16#A0). +-define(RC_SUBSCRIPTION_IDENTIFIERS_NOT_SUPPORTED, 16#A1). +-define(RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED, 16#A2). + +%%-------------------------------------------------------------------- +%% Maximum MQTT Packet Length +%%-------------------------------------------------------------------- + +-define(MAX_PACKET_SIZE, 16#fffffff). + +%%-------------------------------------------------------------------- +%% MQTT Frame Mask +%%-------------------------------------------------------------------- + +-define(HIGHBIT, 2#10000000). +-define(LOWBITS, 2#01111111). + +%%-------------------------------------------------------------------- +%% MQTT Packet Fixed Header +%%-------------------------------------------------------------------- + +-record(mqtt_packet_header, { + type = ?RESERVED, + dup = false, + qos = ?QOS_0, + retain = false + }). + +%%-------------------------------------------------------------------- +%% MQTT Packets +%%-------------------------------------------------------------------- + +-define(DEFAULT_SUBOPTS, #{rh => 0, %% Retain Handling + rap => 0, %% Retain as Publish + nl => 0, %% No Local + qos => 0, %% QoS + rc => 0 %% Reason Code + }). + +-record(mqtt_packet_connect, { + proto_name = <<"MQTT">>, + proto_ver = ?MQTT_PROTO_V4, + is_bridge = false, + clean_start = true, + will_flag = false, + will_qos = ?QOS_0, + will_retain = false, + keepalive = 0, + properties = undefined, + client_id = <<>>, + will_props = undefined, + will_topic = undefined, + will_payload = undefined, + username = undefined, + password = undefined + }). + +-record(mqtt_packet_connack, { + ack_flags, + reason_code, + properties + }). + +-record(mqtt_packet_publish, { + topic_name, + packet_id, + properties + }). + +-record(mqtt_packet_puback, { + packet_id, + reason_code, + properties + }). + +-record(mqtt_packet_subscribe, { + packet_id, + properties, + topic_filters + }). + +-record(mqtt_packet_suback, { + packet_id, + properties, + reason_codes + }). + +-record(mqtt_packet_unsubscribe, { + packet_id, + properties, + topic_filters + }). + +-record(mqtt_packet_unsuback, { + packet_id, + properties, + reason_codes + }). + +-record(mqtt_packet_disconnect, { + reason_code, + properties + }). + +-record(mqtt_packet_auth, { + reason_code, + properties + }). + +%%-------------------------------------------------------------------- +%% MQTT Control Packet +%%-------------------------------------------------------------------- + +-record(mqtt_packet, { + header :: #mqtt_packet_header{}, + variable :: #mqtt_packet_connect{} + | #mqtt_packet_connack{} + | #mqtt_packet_publish{} + | #mqtt_packet_puback{} + | #mqtt_packet_subscribe{} + | #mqtt_packet_suback{} + | #mqtt_packet_unsubscribe{} + | #mqtt_packet_unsuback{} + | #mqtt_packet_disconnect{} + | #mqtt_packet_auth{} + | pos_integer() + | undefined, + payload :: binary() | undefined + }). + +%%-------------------------------------------------------------------- +%% MQTT Packet Match +%%-------------------------------------------------------------------- + +-define(CONNECT_PACKET(Var), + #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT}, + variable = Var}). + +-define(CONNACK_PACKET(ReasonCode), + #mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, + variable = #mqtt_packet_connack{ack_flags = 0, + reason_code = ReasonCode} + }). + +-define(CONNACK_PACKET(ReasonCode, SessPresent), + #mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, + variable = #mqtt_packet_connack{ack_flags = SessPresent, + reason_code = ReasonCode} + }). + +-define(CONNACK_PACKET(ReasonCode, SessPresent, Properties), + #mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, + variable = #mqtt_packet_connack{ack_flags = SessPresent, + reason_code = ReasonCode, + properties = Properties} + }). + +-define(AUTH_PACKET(), + #mqtt_packet{header = #mqtt_packet_header{type = ?AUTH}, + variable = #mqtt_packet_auth{reason_code = 0} + }). + +-define(AUTH_PACKET(ReasonCode), + #mqtt_packet{header = #mqtt_packet_header{type = ?AUTH}, + variable = #mqtt_packet_auth{reason_code = ReasonCode} + }). + +-define(AUTH_PACKET(ReasonCode, Properties), + #mqtt_packet{header = #mqtt_packet_header{type = ?AUTH}, + variable = #mqtt_packet_auth{reason_code = ReasonCode, + properties = Properties} + }). + +-define(PUBLISH_PACKET(QoS), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, qos = QoS}}). + +-define(PUBLISH_PACKET(QoS, PacketId), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + qos = QoS}, + variable = #mqtt_packet_publish{packet_id = PacketId} + }). + +-define(PUBLISH_PACKET(QoS, Topic, PacketId, Payload), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + qos = QoS}, + variable = #mqtt_packet_publish{topic_name = Topic, + packet_id = PacketId}, + payload = Payload + }). + +-define(PUBLISH_PACKET(QoS, Topic, PacketId, Properties, Payload), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + qos = QoS}, + variable = #mqtt_packet_publish{topic_name = Topic, + packet_id = PacketId, + properties = Properties}, + payload = Payload + }). + +-define(PUBACK_PACKET(PacketId), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBACK}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = 0} + }). + +-define(PUBACK_PACKET(PacketId, ReasonCode), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBACK}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode} + }). + +-define(PUBACK_PACKET(PacketId, ReasonCode, Properties), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBACK}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode, + properties = Properties} + }). + +-define(PUBREC_PACKET(PacketId), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREC}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = 0} + }). + +-define(PUBREC_PACKET(PacketId, ReasonCode), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREC}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode} + }). + +-define(PUBREC_PACKET(PacketId, ReasonCode, Properties), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREC}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode, + properties = Properties} + }). + +-define(PUBREL_PACKET(PacketId), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREL, + qos = ?QOS_1}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = 0} + }). + +-define(PUBREL_PACKET(PacketId, ReasonCode), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREL, + qos = ?QOS_1}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode} + }). + +-define(PUBREL_PACKET(PacketId, ReasonCode, Properties), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREL, + qos = ?QOS_1}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode, + properties = Properties} + }). + +-define(PUBCOMP_PACKET(PacketId), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBCOMP}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = 0} + }). + +-define(PUBCOMP_PACKET(PacketId, ReasonCode), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBCOMP}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode} + }). + +-define(PUBCOMP_PACKET(PacketId, ReasonCode, Properties), + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBCOMP}, + variable = #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode, + properties = Properties} + }). + +-define(SUBSCRIBE_PACKET(PacketId, TopicFilters), + #mqtt_packet{header = #mqtt_packet_header{type = ?SUBSCRIBE, + qos = ?QOS_1}, + variable = #mqtt_packet_subscribe{packet_id = PacketId, + topic_filters = TopicFilters} + }). + +-define(SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), + #mqtt_packet{header = #mqtt_packet_header{type = ?SUBSCRIBE, + qos = ?QOS_1}, + variable = #mqtt_packet_subscribe{packet_id = PacketId, + properties = Properties, + topic_filters = TopicFilters} + }). + +-define(SUBACK_PACKET(PacketId, ReasonCodes), + #mqtt_packet{header = #mqtt_packet_header{type = ?SUBACK}, + variable = #mqtt_packet_suback{packet_id = PacketId, + reason_codes = ReasonCodes} + }). + +-define(SUBACK_PACKET(PacketId, Properties, ReasonCodes), + #mqtt_packet{header = #mqtt_packet_header{type = ?SUBACK}, + variable = #mqtt_packet_suback{packet_id = PacketId, + properties = Properties, + reason_codes = ReasonCodes} + }). + +-define(UNSUBSCRIBE_PACKET(PacketId, TopicFilters), + #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBSCRIBE, + qos = ?QOS_1}, + variable = #mqtt_packet_unsubscribe{packet_id = PacketId, + topic_filters = TopicFilters} + }). + +-define(UNSUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), + #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBSCRIBE, + qos = ?QOS_1}, + variable = #mqtt_packet_unsubscribe{packet_id = PacketId, + properties = Properties, + topic_filters = TopicFilters} + }). + +-define(UNSUBACK_PACKET(PacketId), + #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBACK}, + variable = #mqtt_packet_unsuback{packet_id = PacketId} + }). + +-define(UNSUBACK_PACKET(PacketId, ReasonCodes), + #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBACK}, + variable = #mqtt_packet_unsuback{packet_id = PacketId, + reason_codes = ReasonCodes} + }). + +-define(UNSUBACK_PACKET(PacketId, Properties, ReasonCodes), + #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBACK}, + variable = #mqtt_packet_unsuback{packet_id = PacketId, + properties = Properties, + reason_codes = ReasonCodes} + }). + +-define(DISCONNECT_PACKET(), + #mqtt_packet{header = #mqtt_packet_header{type = ?DISCONNECT}, + variable = #mqtt_packet_disconnect{reason_code = 0} + }). + +-define(DISCONNECT_PACKET(ReasonCode), + #mqtt_packet{header = #mqtt_packet_header{type = ?DISCONNECT}, + variable = #mqtt_packet_disconnect{reason_code = ReasonCode} + }). + +-define(DISCONNECT_PACKET(ReasonCode, Properties), + #mqtt_packet{header = #mqtt_packet_header{type = ?DISCONNECT}, + variable = #mqtt_packet_disconnect{reason_code = ReasonCode, + properties = Properties} + }). + +-define(PACKET(Type), #mqtt_packet{header = #mqtt_packet_header{type = Type}}). + +-define(SHARE, "$share"). +-define(SHARE(Group, Topic), emqx_topic:join([<>, Group, Topic])). +-define(IS_SHARE(Topic), case Topic of <> -> true; _ -> false end). + +-endif. diff --git a/include/logger.hrl b/include/logger.hrl new file mode 100644 index 000000000..db93d7f56 --- /dev/null +++ b/include/logger.hrl @@ -0,0 +1,23 @@ +%%-------------------------------------------------------------------- +%% Logs with a header prefixed to the log message. +%% And the log args are puted into report_cb for lazy evaluation. +%%-------------------------------------------------------------------- +-ifdef(LOG_HEADER). +%% with header +-define(LOG(Level, Format, Args), + begin + (logger:log(Level,#{},#{report_cb => + fun(_) -> + {?LOG_HEADER ++ " "++ (Format), (Args)} + end})) + end). +-else. +%% without header +-define(LOG(Level, Format, Args), + begin + (logger:log(Level,#{},#{report_cb => + fun(_) -> + {(Format), (Args)} + end})) + end). +-endif. \ No newline at end of file diff --git a/priv/emq.schema b/priv/emq.schema deleted file mode 100644 index 6ddf9f2a7..000000000 --- a/priv/emq.schema +++ /dev/null @@ -1,1509 +0,0 @@ -%%-*- mode: erlang -*- -%% EMQ R2.3 config mapping - -%%-------------------------------------------------------------------- -%% Cluster -%%-------------------------------------------------------------------- - -%% @doc Cluster name -{mapping, "cluster.name", "ekka.cluster_name", [ - {default, emqcl}, - {datatype, atom} -]}. - -%% @doc Cluster discovery -{mapping, "cluster.discovery", "ekka.cluster_discovery", [ - {default, manual}, - {datatype, atom} -]}. - -%% @doc Clean down node from the cluster -{mapping, "cluster.autoclean", "ekka.cluster_autoclean", [ - {datatype, {duration, ms}} -]}. - -%% @doc Cluster autoheal -{mapping, "cluster.autoheal", "ekka.cluster_autoheal", [ - {datatype, flag}, - {default, off} -]}. - -%%-------------------------------------------------------------------- -%% Cluster by static node list - -{mapping, "cluster.static.seeds", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -%%-------------------------------------------------------------------- -%% Cluster by UDP Multicast - -{mapping, "cluster.mcast.addr", "ekka.cluster_discovery", [ - {default, "239.192.0.1"}, - {datatype, string} -]}. - -{mapping, "cluster.mcast.ports", "ekka.cluster_discovery", [ - {default, "4369"}, - {datatype, string} -]}. - -{mapping, "cluster.mcast.iface", "ekka.cluster_discovery", [ - {datatype, string}, - {default, "0.0.0.0"} -]}. - -{mapping, "cluster.mcast.ttl", "ekka.cluster_discovery", [ - {datatype, integer}, - {default, 255} -]}. - -{mapping, "cluster.mcast.loop", "ekka.cluster_discovery", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "cluster.mcast.sndbuf", "ekka.cluster_discovery", [ - {datatype, bytesize}, - {default, "16KB"} -]}. - -{mapping, "cluster.mcast.recbuf", "ekka.cluster_discovery", [ - {datatype, bytesize}, - {default, "16KB"} -]}. - -{mapping, "cluster.mcast.buffer", "ekka.cluster_discovery", [ - {datatype, bytesize}, - {default, "32KB"} -]}. - -%%-------------------------------------------------------------------- -%% Cluster by DNS A Record - -{mapping, "cluster.dns.name", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.dns.app", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -%%-------------------------------------------------------------------- -%% Cluster using etcd - -{mapping, "cluster.etcd.server", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.etcd.prefix", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.etcd.node_ttl", "ekka.cluster_discovery", [ - {datatype, {duration, ms}}, - {default, "1m"} -]}. - -%%-------------------------------------------------------------------- -%% Cluster on K8s - -{mapping, "cluster.k8s.apiserver", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.k8s.service_name", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.k8s.address_type", "ekka.cluster_discovery", [ - {datatype, {enum, [ip, dns]}} -]}. - -{mapping, "cluster.k8s.app_name", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{mapping, "cluster.k8s.namespace", "ekka.cluster_discovery", [ - {datatype, string} -]}. - -{translation, "ekka.cluster_discovery", fun(Conf) -> - Strategy = cuttlefish:conf_get("cluster.discovery", Conf), - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - IpPort = fun(S) -> - [Addr, Port] = string:tokens(S, ":"), - {ok, Ip} = inet:parse_address(Addr), - {Ip, Port} - end, - Options = fun(static) -> - [{seeds, [list_to_atom(S) || S <- string:tokens(cuttlefish:conf_get("cluster.static.seeds", Conf, ""), ",")]}]; - (mcast) -> - {ok, Addr} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.addr", Conf)), - {ok, Iface} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.iface", Conf)), - Ports = [list_to_integer(S) || S <- string:tokens(cuttlefish:conf_get("cluster.mcast.ports", Conf), ",")], - [{addr, Addr}, {ports, Ports}, {iface, Iface}, - {ttl, cuttlefish:conf_get("cluster.mcast.ttl", Conf, 1)}, - {loop, cuttlefish:conf_get("cluster.mcast.loop", Conf, true)}]; - (dns) -> - [{name, cuttlefish:conf_get("cluster.dns.name", Conf)}, - {app, cuttlefish:conf_get("cluster.dns.app", Conf)}]; - (etcd) -> - [{server, string:tokens(cuttlefish:conf_get("cluster.etcd.server", Conf), ",")}, - {prefix, cuttlefish:conf_get("cluster.etcd.prefix", Conf, "emqcl")}, - {node_ttl, cuttlefish:conf_get("cluster.etcd.node_ttl", Conf, 60)}]; - (k8s) -> - [{apiserver, cuttlefish:conf_get("cluster.k8s.apiserver", Conf)}, - {service_name, cuttlefish:conf_get("cluster.k8s.service_name", Conf)}, - {address_type, cuttlefish:conf_get("cluster.k8s.address_type", Conf, ip)}, - {app_name, cuttlefish:conf_get("cluster.k8s.app_name", Conf)}, - {namespace, cuttlefish:conf_get("cluster.k8s.namespace", Conf)}]; - (manual) -> - [ ] - end, - {Strategy, Filter(Options(Strategy))} -end}. - -%%-------------------------------------------------------------------- -%% Erlang Node -%%-------------------------------------------------------------------- - -%% @doc Erlang node name -{mapping, "node.name", "vm_args.-name", [ - {default, "emq@127.0.0.1"} -]}. - -%% @doc The erlang distributed protocol -{mapping, "node.proto_dist", "vm_args.-proto_dist", [ - {default, "inet_tcp"}, - {datatype, {enum, [inet_tcp, inet6_tcp, inet_tls]}}, - hidden -]}. - -%% @doc Specify SSL Options in the file if using SSL for erlang distribution -{mapping, "node.ssl_dist_optfile", "vm_args.-ssl_dist_optfile", [ - {datatype, string}, - hidden -]}. - -%% @doc Secret cookie for distributed erlang node -{mapping, "node.cookie", "vm_args.-setcookie", [ - {default, "emqsecretcookie"} -]}. - -%% @doc SMP Support -{mapping, "node.smp", "vm_args.-smp", [ - {default, auto}, - {datatype, {enum, [enable, auto, disable]}}, - hidden -]}. - -%% @doc http://erlang.org/doc/man/heart.html -{mapping, "node.heartbeat", "vm_args.-heart", [ - {datatype, flag}, - hidden -]}. - -{translation, "vm_args.-heart", fun(Conf) -> - case cuttlefish:conf_get("node.heartbeat", Conf) of - true -> ""; - false -> cuttlefish:invalid("should be 'on' or comment the line!") - end -end}. - -%% @doc Enable Kernel Poll -{mapping, "node.kernel_poll", "vm_args.+K", [ - {default, on}, - {datatype, flag}, - hidden -]}. - -%% @doc More information at: http://erlang.org/doc/man/erl.html -{mapping, "node.async_threads", "vm_args.+A", [ - {default, 64}, - {datatype, integer}, - {validators, ["range:0-1024"]} -]}. - -%% @doc Erlang Process Limit -{mapping, "node.process_limit", "vm_args.+P", [ - {datatype, integer}, - {default, 256000}, - hidden -]}. - -%% Note: OTP R15 and earlier uses -env ERL_MAX_PORTS, R16+ uses +Q -%% @doc The number of concurrent ports/sockets -%% Valid range is 1024-134217727 -{mapping, "node.max_ports", - cuttlefish:otp("R16", "vm_args.+Q", "vm_args.-env ERL_MAX_PORTS"), [ - {default, 262144}, - {datatype, integer}, - {validators, ["range4ports"]} -]}. - -{validator, "range4ports", "must be 1024 to 134217727", - fun(X) -> X >= 1024 andalso X =< 134217727 end}. - -%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl -{mapping, "node.dist_buffer_size", "vm_args.+zdbbl", [ - {datatype, bytesize}, - {commented, "32MB"}, - hidden, - {validators, ["zdbbl_range"]} -]}. - -{translation, "vm_args.+zdbbl", - fun(Conf) -> - ZDBBL = cuttlefish:conf_get("node.dist_buffer_size", Conf, undefined), - case ZDBBL of - undefined -> undefined; - X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; - _ -> undefined - end - end -}. - -{validator, "zdbbl_range", "must be between 1KB and 2097151KB", - fun(ZDBBL) -> - %% 2097151KB = 2147482624 - ZDBBL >= 1024 andalso ZDBBL =< 2147482624 - end -}. - -%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 -{mapping, "node.fullsweep_after", "vm_args.-env ERL_FULLSWEEP_AFTER", [ - {default, 1000}, - {datatype, integer}, - hidden, - {validators, ["positive_integer"]} -]}. - -{validator, "positive_integer", "must be a positive integer", - fun(X) -> X >= 0 end}. - -%% Note: OTP R15 and earlier uses -env ERL_MAX_ETS_TABLES, -%% R16+ uses +e -%% @doc The ETS table limit -{mapping, "node.max_ets_tables", - cuttlefish:otp("R16", "vm_args.+e", "vm_args.-env ERL_MAX_ETS_TABLES"), [ - {default, 256000}, - {datatype, integer}, - hidden -]}. - -%% @doc Set the location of crash dumps -{mapping, "node.crash_dump", "vm_args.-env ERL_CRASH_DUMP", [ - {default, "{{crash_dump}}"}, - {datatype, file}, - hidden -]}. - -%% @doc http://www.erlang.org/doc/man/kernel_app.html#net_ticktime -{mapping, "node.dist_net_ticktime", "vm_args.-kernel net_ticktime", [ - {commented, 60}, - {datatype, integer}, - hidden -]}. - -%% @doc http://www.erlang.org/doc/man/kernel_app.html -{mapping, "node.dist_listen_min", "kernel.inet_dist_listen_min", [ - {commented, 6369}, - {datatype, integer}, - hidden -]}. - -%% @see node.dist_listen_min -{mapping, "node.dist_listen_max", "kernel.inet_dist_listen_max", [ - {commented, 6369}, - {datatype, integer}, - hidden -]}. - -%%-------------------------------------------------------------------- -%% Log -%%-------------------------------------------------------------------- - -{mapping, "log.dir", "lager.log_dir", [ - {default, "log"}, - {datatype, string} -]}. - -{mapping, "log.console", "lager.handlers", [ - {default, file}, - {datatype, {enum, [off, file, console, both]}} -]}. - -{mapping, "log.console.level", "lager.handlers", [ - {default, info}, - {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} -]}. - -{mapping, "log.console.file", "lager.handlers", [ - {default, "log/console.log"}, - {datatype, file} -]}. - -{mapping, "log.console.size", "lager.handlers", [ - {default, 10485760}, - {datatype, integer} -]}. - -{mapping, "log.console.count", "lager.handlers", [ - {default, 5}, - {datatype, integer} -]}. - -{mapping, "log.info.file", "lager.handlers", [ - {datatype, file} -]}. - -{mapping, "log.info.size", "lager.handlers", [ - {default, 10485760}, - {datatype, integer} -]}. - -{mapping, "log.info.count", "lager.handlers", [ - {default, 5}, - {datatype, integer} -]}. - -{mapping, "log.error.file", "lager.handlers", [ - {default, "log/error.log"}, - {datatype, file} -]}. - -{mapping, "log.error.size", "lager.handlers", [ - {default, 10485760}, - {datatype, integer} -]}. - -{mapping, "log.error.count", "lager.handlers", [ - {default, 5}, - {datatype, integer} -]}. - -{mapping, "log.syslog", "lager.handlers", [ - {default, off}, - {datatype, flag} -]}. - -{mapping, "log.syslog.identity", "lager.handlers", [ - {default, "emqttd"}, - {datatype, string} -]}. - -{mapping, "log.syslog.facility", "lager.handlers", [ - {default, local0}, - {datatype, {enum, [daemon, local0, local1, local2, local3, local4, local5, local6, local7]}} -]}. - -{mapping, "log.syslog.level", "lager.handlers", [ - {default, error}, - {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency]}} -]}. - -{mapping, "log.error.redirect", "lager.error_logger_redirect", [ - {default, on}, - {datatype, flag}, - hidden -]}. - -{mapping, "log.error.messages_per_second", "lager.error_logger_hwm", [ - {default, 1000}, - {datatype, integer}, - hidden -]}. - -{translation, - "lager.handlers", - fun(Conf) -> - ErrorHandler = case cuttlefish:conf_get("log.error.file", Conf, undefined) of - undefined -> []; - ErrorFilename -> [{lager_file_backend, [{file, ErrorFilename}, - {level, error}, - {size, cuttlefish:conf_get("log.error.size", Conf)}, - {date, "$D0"}, - {count, cuttlefish:conf_get("log.error.count", Conf)}]}] - end, - - InfoHandler = case cuttlefish:conf_get("log.info.file", Conf, undefined) of - undefined -> []; - InfoFilename -> [{lager_file_backend, [{file, InfoFilename}, - {level, info}, - {size, cuttlefish:conf_get("log.info.size", Conf)}, - {date, "$D0"}, - {count, cuttlefish:conf_get("log.info.count", Conf)}]}] - end, - - ConsoleLogLevel = cuttlefish:conf_get("log.console.level", Conf), - ConsoleLogFile = cuttlefish:conf_get("log.console.file", Conf), - - ConsoleHandler = {lager_console_backend, ConsoleLogLevel}, - ConsoleFileHandler = {lager_file_backend, [{file, ConsoleLogFile}, - {level, ConsoleLogLevel}, - {size, cuttlefish:conf_get("log.console.size", Conf)}, - {date, "$D0"}, - {count, cuttlefish:conf_get("log.console.count", Conf)}]}, - - ConsoleHandlers = case cuttlefish:conf_get("log.console", Conf) of - off -> []; - file -> [ConsoleFileHandler]; - console -> [ConsoleHandler]; - both -> [ConsoleHandler, ConsoleFileHandler]; - _ -> [] - end, - SyslogHandler = case cuttlefish:conf_get("log.syslog", Conf) of - false -> []; - true -> [{lager_syslog_backend, - [cuttlefish:conf_get("log.syslog.identity", Conf), - cuttlefish:conf_get("log.syslog.facility", Conf), - cuttlefish:conf_get("log.syslog.level", Conf)]}] - end, - ConsoleHandlers ++ ErrorHandler ++ InfoHandler ++ SyslogHandler - end -}. - -{mapping, "log.crash", "lager.crash_log", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "log.crash.file", "lager.crash_log", [ - {default, "log/crash.log"}, - {datatype, file} -]}. - -{translation, - "lager.crash_log", - fun(Conf) -> - case cuttlefish:conf_get("log.crash", Conf) of - false -> undefined; - _ -> - cuttlefish:conf_get("log.crash.file", Conf, "./log/crash.log") - end - end}. - -{mapping, "sasl", "sasl.sasl_error_logger", [ - {default, off}, - {datatype, flag}, - hidden -]}. - -%%-------------------------------------------------------------------- -%% Allow Anonymous and Default ACL -%%-------------------------------------------------------------------- - -%% @doc Allow Anonymous -{mapping, "mqtt.allow_anonymous", "emqttd.allow_anonymous", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc ACL nomatch -{mapping, "mqtt.acl_nomatch", "emqttd.acl_nomatch", [ - {default, allow}, - {datatype, {enum, [allow, deny]}} -]}. - -%% @doc Default ACL File -{mapping, "mqtt.acl_file", "emqttd.acl_file", [ - {datatype, string}, - hidden -]}. - -%% @doc Cache ACL for PUBLISH -{mapping, "mqtt.cache_acl", "emqttd.cache_acl", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -%%-------------------------------------------------------------------- -%% MQTT Protocol -%%-------------------------------------------------------------------- - -%% @doc Set the Max ClientId Length Allowed. -{mapping, "mqtt.max_clientid_len", "emqttd.protocol", [ - {default, 1024}, - {datatype, integer} -]}. - -%% @doc Max Packet Size Allowed, 64K by default. -{mapping, "mqtt.max_packet_size", "emqttd.protocol", [ - {default, "64KB"}, - {datatype, bytesize} -]}. - -%% @doc Keepalive backoff -{mapping, "mqtt.keepalive_backoff", "emqttd.protocol", [ - {default, 1.25}, - {datatype, float} -]}. - -{translation, "emqttd.protocol", fun(Conf) -> - [{max_clientid_len, cuttlefish:conf_get("mqtt.max_clientid_len", Conf)}, - {max_packet_size, cuttlefish:conf_get("mqtt.max_packet_size", Conf)}, - {keepalive_backoff, cuttlefish:conf_get("mqtt.keepalive_backoff", Conf)}] -end}. - -{mapping, "mqtt.websocket_protocol_header", "emqttd.websocket_protocol_header", [ - {default, on}, - {datatype, flag} -]}. - -%%-------------------------------------------------------------------- -%% MQTT Connection -%%-------------------------------------------------------------------- - -%% @doc Force the client to GC: integer -{mapping, "mqtt.conn.force_gc_count", "emqttd.conn_force_gc_count", [ - {datatype, integer} -]}. - -%%-------------------------------------------------------------------- -%% MQTT Client -%%-------------------------------------------------------------------- - -%% @doc Max Publish Rate of Message -{mapping, "mqtt.client.max_publish_rate", "emqttd.client", [ - {default, 0}, - {datatype, integer} -]}. - -%% @doc Client Idle Timeout. -{mapping, "mqtt.client.idle_timeout", "emqttd.client", [ - {default, "30s"}, - {datatype, {duration, ms}} -]}. - -%% @doc Enable Stats of Client. -{mapping, "mqtt.client.enable_stats", "emqttd.client", [ - {default, off}, - {datatype, flag} -]}. - -{translation, "emqttd.client", fun(Conf) -> - [{max_publish_rate, cuttlefish:conf_get("mqtt.client.max_publish_rate", Conf)}, - {client_idle_timeout, cuttlefish:conf_get("mqtt.client.idle_timeout", Conf)}, - {client_enable_stats, cuttlefish:conf_get("mqtt.client.enable_stats", Conf)}] -end}. - -%%-------------------------------------------------------------------- -%% MQTT Session -%%-------------------------------------------------------------------- - -%% @doc Max Number of Subscriptions Allowed -{mapping, "mqtt.session.max_subscriptions", "emqttd.session", [ - {default, 0}, - {datatype, integer} -]}. - -%% @doc Upgrade QoS? -{mapping, "mqtt.session.upgrade_qos", "emqttd.session", [ - {default, off}, - {datatype, flag} -]}. - -%% @doc Max number of QoS 1 and 2 messages that can be “inflight” at one time. -%% 0 means no limit -{mapping, "mqtt.session.max_inflight", "emqttd.session", [ - {default, 100}, - {datatype, integer} -]}. - -%% @doc Retry interval for redelivering QoS1/2 messages. -{mapping, "mqtt.session.retry_interval", "emqttd.session", [ - {default, "20s"}, - {datatype, {duration, ms}} -]}. - -%% @doc Max Packets that Awaiting PUBREL, 0 means no limit -{mapping, "mqtt.session.max_awaiting_rel", "emqttd.session", [ - {default, 0}, - {datatype, integer} -]}. - -%% @doc Awaiting PUBREL Timeout -{mapping, "mqtt.session.await_rel_timeout", "emqttd.session", [ - {default, "20s"}, - {datatype, {duration, ms}} -]}. - -%% @doc Enable Stats -{mapping, "mqtt.session.enable_stats", "emqttd.session", [ - {default, off}, - {datatype, flag} -]}. - -%% @doc Session Expiry Interval -{mapping, "mqtt.session.expiry_interval", "emqttd.session", [ - {default, "2h"}, - {datatype, {duration, ms}} -]}. - -%% @doc Ignore message from self publish -{mapping, "mqtt.session.ignore_loop_deliver", "emqttd.session", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -{translation, "emqttd.session", fun(Conf) -> - [{max_subscriptions, cuttlefish:conf_get("mqtt.session.max_subscriptions", Conf)}, - {upgrade_qos, cuttlefish:conf_get("mqtt.session.upgrade_qos", Conf)}, - {max_inflight, cuttlefish:conf_get("mqtt.session.max_inflight", Conf)}, - {retry_interval, cuttlefish:conf_get("mqtt.session.retry_interval", Conf)}, - {max_awaiting_rel, cuttlefish:conf_get("mqtt.session.max_awaiting_rel", Conf)}, - {await_rel_timeout, cuttlefish:conf_get("mqtt.session.await_rel_timeout", Conf)}, - {enable_stats, cuttlefish:conf_get("mqtt.session.enable_stats", Conf)}, - {expiry_interval, cuttlefish:conf_get("mqtt.session.expiry_interval", Conf)}, - {ignore_loop_deliver, cuttlefish:conf_get("mqtt.session.ignore_loop_deliver", Conf)}] -end}. - -%%-------------------------------------------------------------------- -%% MQTT MQueue -%%-------------------------------------------------------------------- - -%% @doc Type: simple | priority -{mapping, "mqtt.mqueue.type", "emqttd.mqueue", [ - {default, simple}, - {datatype, atom} -]}. - -%% @doc Topic Priority: 0~255, Default is 0 -{mapping, "mqtt.mqueue.priority", "emqttd.mqueue", [ - {default, ""}, - {datatype, string} -]}. - -%% @doc Max queue length. Enqueued messages when persistent client disconnected, or inflight window is full. 0 means no limit. -{mapping, "mqtt.mqueue.max_length", "emqttd.mqueue", [ - {default, 0}, - {datatype, integer} -]}. - -%% @doc Low-water mark of queued messages -{mapping, "mqtt.mqueue.low_watermark", "emqttd.mqueue", [ - {default, "20%"}, - {datatype, {percent, float}} -]}. - -%% @doc High-water mark of queued messages -{mapping, "mqtt.mqueue.high_watermark", "emqttd.mqueue", [ - {default, "60%"}, - {datatype, {percent, float}} -]}. - -%% @doc Queue Qos0 messages? -{mapping, "mqtt.mqueue.store_qos0", "emqttd.mqueue", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{translation, "emqttd.mqueue", fun(Conf) -> - Opts = [{type, cuttlefish:conf_get("mqtt.mqueue.type", Conf, simple)}, - {max_length, cuttlefish:conf_get("mqtt.mqueue.max_length", Conf)}, - {low_watermark, cuttlefish:conf_get("mqtt.mqueue.low_watermark", Conf)}, - {high_watermark, cuttlefish:conf_get("mqtt.mqueue.high_watermark", Conf)}, - {store_qos0, cuttlefish:conf_get("mqtt.mqueue.store_qos0", Conf)}], - case cuttlefish:conf_get("mqtt.mqueue.priority", Conf) of - undefined -> Opts; - V -> [{priority, - [begin [T, P] = string:tokens(S, "="), - {T, list_to_integer(P)} - end || S <- string:tokens(V, ",")]} | Opts] - end -end}. - -%%-------------------------------------------------------------------- -%% MQTT Broker -%%-------------------------------------------------------------------- - -{mapping, "mqtt.broker.sys_interval", "emqttd.broker_sys_interval", [ - {datatype, {duration, ms}}, - {default, "1m"} -]}. - -%%-------------------------------------------------------------------- -%% MQTT PubSub -%%-------------------------------------------------------------------- - -{mapping, "mqtt.pubsub.pool_size", "emqttd.pubsub", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "mqtt.pubsub.async", "emqttd.pubsub", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{translation, "emqttd.pubsub", fun(Conf) -> - [{pool_size, cuttlefish:conf_get("mqtt.pubsub.pool_size", Conf)}, - {async, cuttlefish:conf_get("mqtt.pubsub.async", Conf)}] -end}. - -%%-------------------------------------------------------------------- -%% MQTT Bridge -%%-------------------------------------------------------------------- - -{mapping, "mqtt.bridge.max_queue_len", "emqttd.bridge", [ - {default, 10000}, - {datatype, integer} -]}. - -{mapping, "mqtt.bridge.ping_down_interval", "emqttd.bridge", [ - {datatype, {duration, ms}}, - {default, "1s"} -]}. - -{translation, "emqttd.bridge", fun(Conf) -> - [{max_queue_len, cuttlefish:conf_get("mqtt.bridge.max_queue_len", Conf)}, - {ping_down_interval, cuttlefish:conf_get("mqtt.bridge.ping_down_interval", Conf)}] -end}. - -%%------------------------------------------------------------------- -%% MQTT Plugins -%%------------------------------------------------------------------- - -{mapping, "mqtt.plugins.etc_dir", "emqttd.plugins_etc_dir", [ - {datatype, string} -]}. - -{mapping, "mqtt.plugins.loaded_file", "emqttd.plugins_loaded_file", [ - {datatype, string} -]}. - -%%-------------------------------------------------------------------- -%% MQTT Listeners -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% TCP Listeners - -{mapping, "listener.tcp.$name", "emqttd.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.tcp.$name.acceptors", "emqttd.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.tcp.$name.max_clients", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.tcp.$name.zone", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.tcp.$name.mountpoint", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.tcp.$name.rate_limit", "emqttd.listeners", [ - {default, undefined}, - {datatype, string} -]}. - -{mapping, "listener.tcp.$name.access.$id", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.tcp.$name.proxy_protocol", "emqttd.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.tcp.$name.proxy_protocol_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}} -]}. - -{mapping, "listener.tcp.$name.peer_cert_as_username", "emqttd.listeners", [ - {datatype, {enum, [cn, dn]}} -]}. - -{mapping, "listener.tcp.$name.backlog", "emqttd.listeners", [ - {datatype, integer}, - {default, 1024} -]}. - -{mapping, "listener.tcp.$name.send_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.tcp.$name.send_timeout_close", "emqttd.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.tcp.$name.recbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.tcp.$name.sndbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.tcp.$name.buffer", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.tcp.$name.tune_buffer", "emqttd.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.tcp.$name.nodelay", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.tcp.$name.reuseaddr", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -%%-------------------------------------------------------------------- -%% SSL Listeners - -{mapping, "listener.ssl.$name", "emqttd.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.ssl.$name.acceptors", "emqttd.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.max_clients", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.zone", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.mountpoint", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.rate_limit", "emqttd.listeners", [ - {default, undefined}, - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.access.$id", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.proxy_protocol", "emqttd.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.ssl.$name.proxy_protocol_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}} -]}. - -{mapping, "listener.ssl.$name.backlog", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.ssl.$name.send_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.ssl.$name.send_timeout_close", "emqttd.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.ssl.$name.recbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ssl.$name.sndbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ssl.$name.buffer", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ssl.$name.tune_buffer", "emqttd.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.ssl.$name.nodelay", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.ssl.$name.reuseaddr", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.ssl.$name.tls_versions", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.ciphers", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.handshake_timeout", "emqttd.listeners", [ - {default, "15s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "listener.ssl.$name.dhfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.keyfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.certfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.cacertfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ssl.$name.verify", "emqttd.listeners", [ - {datatype, atom} -]}. - -{mapping, "listener.ssl.$name.fail_if_no_peer_cert", "emqttd.listeners", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "listener.ssl.$name.secure_renegotiate", "emqttd.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.ssl.$name.reuse_sessions", "emqttd.listeners", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "listener.ssl.$name.honor_cipher_order", "emqttd.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.ssl.$name.peer_cert_as_username", "emqttd.listeners", [ - {datatype, {enum, [cn, dn]}} -]}. - -%%-------------------------------------------------------------------- -%% MQTT/WebSocket Listeners - -{mapping, "listener.ws.$name", "emqttd.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.ws.$name.acceptors", "emqttd.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.ws.$name.max_clients", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.ws.$name.rate_limit", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ws.$name.zone", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ws.$name.mountpoint", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ws.$name.access.$id", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.ws.$name.proxy_address_header", "emqttd.listeners", [ - {datatype, string}, - hidden -]}. - -{mapping, "listener.ws.$name.proxy_port_header", "emqttd.listeners", [ - {datatype, string}, - hidden -]}. - -{mapping, "listener.ws.$name.proxy_protocol", "emqttd.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.ws.$name.proxy_protocol_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}} -]}. - -{mapping, "listener.ws.$name.backlog", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.ws.$name.send_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.ws.$name.send_timeout_close", "emqttd.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.ws.$name.recbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ws.$name.sndbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ws.$name.buffer", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.ws.$name.tune_buffer", "emqttd.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.ws.$name.nodelay", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.ws.$name.reuseaddr", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -%%-------------------------------------------------------------------- -%% MQTT/WebSocket/SSL Listeners - -{mapping, "listener.wss.$name", "emqttd.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.wss.$name.acceptors", "emqttd.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.max_clients", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.zone", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.mountpoint", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.rate_limit", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.access.$id", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.proxy_address_header", "emqttd.listeners", [ - {datatype, string}, - hidden -]}. - -{mapping, "listener.wss.$name.proxy_port_header", "emqttd.listeners", [ - {datatype, string}, - hidden -]}. - -{mapping, "listener.wss.$name.proxy_protocol", "emqttd.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.wss.$name.proxy_protocol_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}} -]}. - -{mapping, "listener.wss.$name.backlog", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.wss.$name.send_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.wss.$name.send_timeout_close", "emqttd.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.wss.$name.recbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.wss.$name.sndbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.wss.$name.buffer", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.wss.$name.tune_buffer", "emqttd.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.wss.$name.nodelay", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.wss.$name.reuseaddr", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.wss.$name.tls_versions", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.ciphers", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.handshake_timeout", "emqttd.listeners", [ - {default, "15s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "listener.wss.$name.keyfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.certfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.cacertfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.wss.$name.verify", "emqttd.listeners", [ - {datatype, atom} -]}. - -{mapping, "listener.wss.$name.fail_if_no_peer_cert", "emqttd.listeners", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "listener.wss.$name.secure_renegotiate", "emqttd.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.wss.$name.reuse_sessions", "emqttd.listeners", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "listener.wss.$name.honor_cipher_order", "emqttd.listeners", [ - {datatype, flag} -]}. - -{mapping, "listener.wss.$name.peer_cert_as_username", "emqttd.listeners", [ - {datatype, {enum, [cn, dn]}} -]}. - -{translation, "emqttd.listeners", fun(Conf) -> - - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - - Atom = fun(undefined) -> undefined; (S) -> list_to_atom(S) end, - - Access = fun(S) -> - [A, CIDR] = string:tokens(S, " "), - {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} - end, - - AccOpts = fun(Prefix) -> - case cuttlefish_variable:filter_by_prefix(Prefix ++ ".access", Conf) of - [] -> []; - Rules -> [{access, [Access(Rule) || {_, Rule} <- Rules]}] - end - end, - - MountPoint = fun(undefined) -> undefined; (S) -> list_to_binary(S) end, - - ConnOpts = fun(Prefix) -> - Filter([{zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))}, - {rate_limit, cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined)}, - {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)}, - {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)}, - {mountpoint, MountPoint(cuttlefish:conf_get(Prefix ++ ".mountpoint", Conf, undefined))}, - {peer_cert_as_username, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_username", Conf, undefined)}, - {proxy_port_header, cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, undefined)}, - {proxy_address_header, cuttlefish:conf_get(Prefix ++ ".proxy_address_header", Conf, undefined)}]) - end, - - LisOpts = fun(Prefix) -> - Filter([{acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, - {max_clients, cuttlefish:conf_get(Prefix ++ ".max_clients", Conf)}, - {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)} | AccOpts(Prefix)]) - end, - TcpOpts = fun(Prefix) -> - Filter([{backlog, cuttlefish:conf_get(Prefix ++ ".backlog", Conf, undefined)}, - {send_timeout, cuttlefish:conf_get(Prefix ++ ".send_timeout", Conf, undefined)}, - {send_timeout_close, cuttlefish:conf_get(Prefix ++ ".send_timeout_close", Conf, undefined)}, - {recbuf, cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)}, - {sndbuf, cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)}, - {buffer, cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)}, - {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)}, - {reuseaddr, cuttlefish:conf_get(Prefix ++ ".reuseaddr", Conf, true)}]) - end, - - SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - - SslOpts = fun(Prefix) -> - Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of - undefined -> undefined; - L -> [list_to_atom(V) || V <- L] - end, - Filter([{versions, Versions}, - {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))}, - {handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf, undefined)}, - {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, - {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, - {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, - {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, - {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, - {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) - end, - - TcpListeners = fun(Type, Name) -> - Prefix = string:join(["listener", Type, Name], "."), - case cuttlefish:conf_get(Prefix, Conf, undefined) of - undefined -> - []; - ListenOn -> - [{Atom(Type), ListenOn, [{connopts, ConnOpts(Prefix)}, - {sockopts, TcpOpts(Prefix)} | LisOpts(Prefix)]}] - end - end, - - SslListeners = fun(Type, Name) -> - Prefix = string:join(["listener", Type, Name], "."), - case cuttlefish:conf_get(Prefix, Conf, undefined) of - undefined -> - []; - ListenOn -> - [{Atom(Type), ListenOn, [{connopts, ConnOpts(Prefix)}, - {sockopts, TcpOpts(Prefix)}, - {sslopts, SslOpts(Prefix)} | LisOpts(Prefix)]}] - end - end, - - ApiListeners = fun(Type, Name) -> - Prefix = string:join(["listener", Type, Name], "."), - case cuttlefish:conf_get(Prefix, Conf, undefined) of - undefined -> - []; - ListenOn -> - SslOpts1 = case SslOpts(Prefix) of - [] -> []; - SslOpts0 -> [{sslopts, SslOpts0}] - end, - [{Atom(Type), ListenOn, [{connopts, ConnOpts(Prefix)}, - {sockopts, TcpOpts(Prefix)}| LisOpts(Prefix)] ++ SslOpts1}] - end - end, - - - lists:flatten([TcpListeners(Type, Name) || {["listener", Type, Name], ListenOn} - <- cuttlefish_variable:filter_by_prefix("listener.tcp", Conf) - ++ cuttlefish_variable:filter_by_prefix("listener.ws", Conf)] - ++ - [SslListeners(Type, Name) || {["listener", Type, Name], ListenOn} - <- cuttlefish_variable:filter_by_prefix("listener.ssl", Conf) - ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)] - ++ - [ApiListeners(Type, Name) || {["listener", Type, Name], ListenOn} - <- cuttlefish_variable:filter_by_prefix("listener.api", Conf)]) -end}. - -%%-------------------------------------------------------------------- -%% MQTT REST API Listeners - -{mapping, "listener.api.$name", "emqttd.listeners", [ - {datatype, [integer, ip]} -]}. - -{mapping, "listener.api.$name.acceptors", "emqttd.listeners", [ - {default, 8}, - {datatype, integer} -]}. - -{mapping, "listener.api.$name.max_clients", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.api.$name.rate_limit", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.api.$name.access.$id", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.api.$name.backlog", "emqttd.listeners", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "listener.api.$name.send_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}}, - {default, "15s"} -]}. - -{mapping, "listener.api.$name.send_timeout_close", "emqttd.listeners", [ - {datatype, flag}, - {default, on} -]}. - -{mapping, "listener.api.$name.recbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.api.$name.sndbuf", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.api.$name.buffer", "emqttd.listeners", [ - {datatype, bytesize}, - hidden -]}. - -{mapping, "listener.api.$name.tune_buffer", "emqttd.listeners", [ - {datatype, flag}, - hidden -]}. - -{mapping, "listener.api.$name.nodelay", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.api.$name.reuseaddr", "emqttd.listeners", [ - {datatype, {enum, [true, false]}}, - hidden -]}. - -{mapping, "listener.api.$name.handshake_timeout", "emqttd.listeners", [ - {datatype, {duration, ms}} -]}. - -{mapping, "listener.api.$name.keyfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.api.$name.certfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.api.$name.cacertfile", "emqttd.listeners", [ - {datatype, string} -]}. - -{mapping, "listener.api.$name.verify", "emqttd.listeners", [ - {datatype, atom} -]}. - -{mapping, "listener.api.$name.fail_if_no_peer_cert", "emqttd.listeners", [ - {datatype, {enum, [true, false]}} -]}. - -%%-------------------------------------------------------------------- -%% System Monitor -%%-------------------------------------------------------------------- - -%% @doc Long GC, don't monitor in production mode for: -%% https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 -{mapping, "sysmon.long_gc", "emqttd.sysmon", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Long Schedule(ms) -{mapping, "sysmon.long_schedule", "emqttd.sysmon", [ - {default, 1000}, - {datatype, integer} -]}. - -%% @doc Large Heap -{mapping, "sysmon.large_heap", "emqttd.sysmon", [ - {default, "8MB"}, - {datatype, bytesize} -]}. - -%% @doc Monitor Busy Port -{mapping, "sysmon.busy_port", "emqttd.sysmon", [ - {default, false}, - {datatype, {enum, [true, false]}} -]}. - -%% @doc Monitor Busy Dist Port -{mapping, "sysmon.busy_dist_port", "emqttd.sysmon", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{translation, "emqttd.sysmon", fun(Conf) -> - [{long_gc, cuttlefish:conf_get("sysmon.long_gc", Conf)}, - {long_schedule, cuttlefish:conf_get("sysmon.long_schedule", Conf)}, - {large_heap, cuttlefish:conf_get("sysmon.large_heap", Conf)}, - {busy_port, cuttlefish:conf_get("sysmon.busy_port", Conf)}, - {busy_dist_port, cuttlefish:conf_get("sysmon.busy_dist_port", Conf)}] -end}. - diff --git a/priv/emqx.schema b/priv/emqx.schema new file mode 100644 index 000000000..f74b53e7f --- /dev/null +++ b/priv/emqx.schema @@ -0,0 +1,1809 @@ +%%-*- mode: erlang -*- +%% EMQ X R3.0 config mapping + +%%-------------------------------------------------------------------- +%% Cluster +%%-------------------------------------------------------------------- + +%% @doc Cluster name +{mapping, "cluster.name", "ekka.cluster_name", [ + {default, emqxcl}, + {datatype, atom} +]}. + +%% @doc Cluster discovery +{mapping, "cluster.discovery", "ekka.cluster_discovery", [ + {default, manual}, + {datatype, atom} +]}. + +%% @doc Clean down node from the cluster +{mapping, "cluster.autoclean", "ekka.cluster_autoclean", [ + {datatype, {duration, ms}} +]}. + +%% @doc Cluster autoheal +{mapping, "cluster.autoheal", "ekka.cluster_autoheal", [ + {datatype, flag}, + {default, off} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by static node list + +{mapping, "cluster.static.seeds", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by UDP Multicast + +{mapping, "cluster.mcast.addr", "ekka.cluster_discovery", [ + {default, "239.192.0.1"}, + {datatype, string} +]}. + +{mapping, "cluster.mcast.ports", "ekka.cluster_discovery", [ + {default, "4369"}, + {datatype, string} +]}. + +{mapping, "cluster.mcast.iface", "ekka.cluster_discovery", [ + {datatype, string}, + {default, "0.0.0.0"} +]}. + +{mapping, "cluster.mcast.ttl", "ekka.cluster_discovery", [ + {datatype, integer}, + {default, 255} +]}. + +{mapping, "cluster.mcast.loop", "ekka.cluster_discovery", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "cluster.mcast.sndbuf", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "16KB"} +]}. + +{mapping, "cluster.mcast.recbuf", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "16KB"} +]}. + +{mapping, "cluster.mcast.buffer", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "32KB"} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by DNS A Record + +{mapping, "cluster.dns.name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.dns.app", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Cluster using etcd + +{mapping, "cluster.etcd.server", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.prefix", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.node_ttl", "ekka.cluster_discovery", [ + {datatype, {duration, ms}}, + {default, "1m"} +]}. + +%%-------------------------------------------------------------------- +%% Cluster on K8s + +{mapping, "cluster.k8s.apiserver", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.service_name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.address_type", "ekka.cluster_discovery", [ + {datatype, {enum, [ip, dns]}} +]}. + +{mapping, "cluster.k8s.app_name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.namespace", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{translation, "ekka.cluster_discovery", fun(Conf) -> + Strategy = cuttlefish:conf_get("cluster.discovery", Conf), + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + IpPort = fun(S) -> + [Addr, Port] = string:tokens(S, ":"), + {ok, Ip} = inet:parse_address(Addr), + {Ip, Port} + end, + Options = fun(static) -> + [{seeds, [list_to_atom(S) || S <- string:tokens(cuttlefish:conf_get("cluster.static.seeds", Conf, ""), ",")]}]; + (mcast) -> + {ok, Addr} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.addr", Conf)), + {ok, Iface} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.iface", Conf)), + Ports = [list_to_integer(S) || S <- string:tokens(cuttlefish:conf_get("cluster.mcast.ports", Conf), ",")], + [{addr, Addr}, {ports, Ports}, {iface, Iface}, + {ttl, cuttlefish:conf_get("cluster.mcast.ttl", Conf, 1)}, + {loop, cuttlefish:conf_get("cluster.mcast.loop", Conf, true)}]; + (dns) -> + [{name, cuttlefish:conf_get("cluster.dns.name", Conf)}, + {app, cuttlefish:conf_get("cluster.dns.app", Conf)}]; + (etcd) -> + [{server, string:tokens(cuttlefish:conf_get("cluster.etcd.server", Conf), ",")}, + {prefix, cuttlefish:conf_get("cluster.etcd.prefix", Conf, "emqcl")}, + {node_ttl, cuttlefish:conf_get("cluster.etcd.node_ttl", Conf, 60)}]; + (k8s) -> + [{apiserver, cuttlefish:conf_get("cluster.k8s.apiserver", Conf)}, + {service_name, cuttlefish:conf_get("cluster.k8s.service_name", Conf)}, + {address_type, cuttlefish:conf_get("cluster.k8s.address_type", Conf, ip)}, + {app_name, cuttlefish:conf_get("cluster.k8s.app_name", Conf)}, + {namespace, cuttlefish:conf_get("cluster.k8s.namespace", Conf)}]; + (manual) -> + [ ] + end, + {Strategy, Filter(Options(Strategy))} +end}. + +%%-------------------------------------------------------------------- +%% Node +%%-------------------------------------------------------------------- + +%% @doc Node name +{mapping, "node.name", "vm_args.-name", [ + {default, "emqx@127.0.0.1"} +]}. + +%% @doc The erlang distributed protocol +{mapping, "node.proto_dist", "vm_args.-proto_dist", [ + %{default, "inet_tcp"}, + {datatype, {enum, [inet_tcp, inet6_tcp, inet_tls]}}, + hidden +]}. + +%% @doc Specify SSL Options in the file if using SSL for erlang distribution +{mapping, "node.ssl_dist_optfile", "vm_args.-ssl_dist_optfile", [ + {datatype, string}, + hidden +]}. + +%% @doc Secret cookie for distributed erlang node +{mapping, "node.cookie", "vm_args.-setcookie", [ + {default, "emqxsecretcookie"} +]}. + +%% @doc http://erlang.org/doc/man/heart.html +{mapping, "node.heartbeat", "vm_args.-heart", [ + {datatype, flag}, + hidden +]}. + +{translation, "vm_args.-heart", fun(Conf) -> + case cuttlefish:conf_get("node.heartbeat", Conf) of + true -> ""; + false -> cuttlefish:invalid("should be 'on' or comment the line!") + end +end}. + +%% @doc More information at: http://erlang.org/doc/man/erl.html +{mapping, "node.async_threads", "vm_args.+A", [ + {default, 64}, + {datatype, integer}, + {validators, ["range:0-1024"]} +]}. + +%% @doc Erlang Process Limit +{mapping, "node.process_limit", "vm_args.+P", [ + {datatype, integer}, + {default, 256000}, + hidden +]}. + +%% Note: OTP R15 and earlier uses -env ERL_MAX_PORTS, R16+ uses +Q +%% @doc The number of concurrent ports/sockets +%% Valid range is 1024-134217727 +{mapping, "node.max_ports", + cuttlefish:otp("R16", "vm_args.+Q", "vm_args.-env ERL_MAX_PORTS"), [ + {default, 262144}, + {datatype, integer}, + {validators, ["range4ports"]} +]}. + +{validator, "range4ports", "must be 1024 to 134217727", + fun(X) -> X >= 1024 andalso X =< 134217727 end}. + +%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl +{mapping, "node.dist_buffer_size", "vm_args.+zdbbl", [ + {datatype, bytesize}, + {commented, "32MB"}, + hidden, + {validators, ["zdbbl_range"]} +]}. + +{translation, "vm_args.+zdbbl", + fun(Conf) -> + ZDBBL = cuttlefish:conf_get("node.dist_buffer_size", Conf, undefined), + case ZDBBL of + undefined -> undefined; + X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; + _ -> undefined + end + end +}. + +{validator, "zdbbl_range", "must be between 1KB and 2097151KB", + fun(ZDBBL) -> + %% 2097151KB = 2147482624 + ZDBBL >= 1024 andalso ZDBBL =< 2147482624 + end +}. + +%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 +{mapping, "node.fullsweep_after", "vm_args.-env ERL_FULLSWEEP_AFTER", [ + {default, 1000}, + {datatype, integer}, + hidden, + {validators, ["positive_integer"]} +]}. + +{validator, "positive_integer", "must be a positive integer", + fun(X) -> X >= 0 end}. + +%% Note: OTP R15 and earlier uses -env ERL_MAX_ETS_TABLES, +%% R16+ uses +e +%% @doc The ETS table limit +{mapping, "node.max_ets_tables", + cuttlefish:otp("R16", "vm_args.+e", "vm_args.-env ERL_MAX_ETS_TABLES"), [ + {default, 256000}, + {datatype, integer}, + hidden +]}. + +%% @doc Set the location of crash dumps +{mapping, "node.crash_dump", "vm_args.-env ERL_CRASH_DUMP", [ + {default, "{{crash_dump}}"}, + {datatype, file}, + hidden +]}. + +%% @doc http://www.erlang.org/doc/man/kernel_app.html#net_ticktime +{mapping, "node.dist_net_ticktime", "vm_args.-kernel net_ticktime", [ + {commented, 60}, + {datatype, integer}, + hidden +]}. + +%% @doc http://www.erlang.org/doc/man/kernel_app.html +{mapping, "node.dist_listen_min", "kernel.inet_dist_listen_min", [ + {commented, 6369}, + {datatype, integer}, + hidden +]}. + +%% @see node.dist_listen_min +{mapping, "node.dist_listen_max", "kernel.inet_dist_listen_max", [ + {commented, 6369}, + {datatype, integer}, + hidden +]}. + +%%-------------------------------------------------------------------- +%% RPC +%%-------------------------------------------------------------------- + +%% RPC server port. +{mapping, "rpc.tcp_server_port", "gen_rpc.tcp_server_port", [ + {default, 5369}, + {datatype, integer} +]}. + +%% Default TCP port for outgoing connections +{mapping, "rpc.tcp_client_port", "gen_rpc.tcp_client_port", [ + {default, 5369}, + {datatype, integer} +]}. + +%% Client connect timeout +{mapping, "rpc.connect_timeout", "gen_rpc.connect_timeout", [ + {default, 5000}, + {datatype, integer} +]}. + +%% Client and Server send timeout +{mapping, "rpc.send_timeout", "gen_rpc.send_timeout", [ + {default, 5000}, + {datatype, integer} +]}. + +%% Authentication timeout +{mapping, "rpc.authentication_timeout", "gen_rpc.authentication_timeout", [ + {default, 5000}, + {datatype, integer} +]}. + +%% Default receive timeout for call() functions +{mapping, "rpc.call_receive_timeout", "gen_rpc.call_receive_timeout", [ + {default, 15000}, + {datatype, integer} +]}. + +%% Socket keepalive configuration +{mapping, "rpc.socket_keepalive_idle", "gen_rpc.socket_keepalive_idle", [ + {default, 7200}, + {datatype, integer} +]}. + +%% Seconds between probes +{mapping, "rpc.socket_keepalive_interval", "gen_rpc.socket_keepalive_interval", [ + {default, 75}, + {datatype, integer} +]}. + +%% Probes lost to close the connection +{mapping, "rpc.socket_keepalive_count", "gen_rpc.socket_keepalive_count", [ + {default, 9}, + {datatype, integer} +]}. + +%%-------------------------------------------------------------------- +%% Log +%%-------------------------------------------------------------------- + +{mapping, "log.to", "kernel.logger", [ + {default, console}, + {datatype, {enum, [off, file, console, both]}} +]}. + +{mapping, "log.level", "kernel.logger", [ + {default, error}, + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} +]}. + +{mapping, "log.primary_level", "emqx.primary_log_level", [ + {default, error}, + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} +]}. + +{mapping, "log.logger_sasl_compatible", "kernel.logger_sasl_compatible", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "log.dir", "kernel.logger", [ + {default, "log"}, + {datatype, string} +]}. + +{mapping, "log.file", "kernel.logger", [ + {default, "emqx.log"}, + {datatype, file} +]}. + +{mapping, "log.rotation.size", "kernel.logger", [ + {default, "10MB"}, + {datatype, bytesize} +]}. + +{mapping, "log.rotation.count", "kernel.logger", [ + {default, 5}, + {datatype, integer} +]}. + +{mapping, "log.$level.file", "kernel.logger", [ + {datatype, file} +]}. + +{mapping, "log.sasl", "sasl.sasl_error_logger", [ + {default, off}, + {datatype, flag}, + hidden +]}. + +{mapping, "log.error_logger", "kernel.error_logger", [ + {default, silent}, + {datatype, {enum, [silent]}}, + hidden +]}. + +{translation, "emqx.primary_log_level", fun(Conf) -> + cuttlefish:conf_get("log.level", Conf) +end}. + +{translation, "kernel.logger", fun(Conf) -> + LogTo = cuttlefish:conf_get("log.to", Conf), + LogLevel = cuttlefish:conf_get("log.level", Conf), + Formatter = {emqx_logger_formatter, + #{template => + [time," [",level,"] ", + {client_id, + [{peername, + [client_id,"@",peername," "], + [client_id, " "]}], + [{peername, + [peername," "], + []}]}, + msg,"\n"]}}, + FileConf = fun(Filename) -> + #{type => wrap, + file => filename:join(cuttlefish:conf_get("log.dir", Conf), Filename), + max_no_files => cuttlefish:conf_get("log.rotation.count", Conf), + max_no_bytes => cuttlefish:conf_get("log.rotation.size", Conf)} + end, + + %% For the default logger that outputs to console + DefaultHandler = + if LogTo =:= console orelse LogTo =:= both -> + [{handler, default, logger_std_h, + #{level => LogLevel, + config => #{type => standard_io}, + formatter => Formatter}}]; + true -> + [{handler, default, undefined}] + end, + + %% For the file logger + FileHandler = + if LogTo =:= file orelse LogTo =:= both -> + [{handler, file, logger_disk_log_h, + #{level => LogLevel, + config => FileConf(cuttlefish:conf_get("log.file", Conf)), + formatter => Formatter, + filesync_repeat_interval => no_repeat}}]; + true -> [] + end, + + %% For creating additional log files for specific log levels. + AdditionalLogFiles = + lists:foldl( + fun({[_, Level, _] = K, Filename}, Acc) when LogTo =:= file; LogTo =:= both -> + case cuttlefish_variable:is_fuzzy_match(K, ["log", "$level", "file"]) of + true -> [{Level, Filename} | Acc]; + false -> Acc + end; + ({_K, _V}, Acc) -> + Acc + end, [], Conf), + AdditionalHandlers = + [{handler, list_to_atom("file_for_"++Level), logger_disk_log_h, + #{level => list_to_atom(Level), + config => FileConf(Filename), + formatter => Formatter, + filesync_repeat_interval => no_repeat}} + || {Level, Filename} <- AdditionalLogFiles], + + DefaultHandler ++ FileHandler ++ AdditionalHandlers +end}. + +%%-------------------------------------------------------------------- +%% Authentication/ACL +%%-------------------------------------------------------------------- + +%% @doc Allow anonymous authentication. +{mapping, "allow_anonymous", "emqx.allow_anonymous", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc ACL nomatch. +{mapping, "acl_nomatch", "emqx.acl_nomatch", [ + {default, deny}, + {datatype, {enum, [allow, deny]}} +]}. + +%% @doc Default ACL file. +{mapping, "acl_file", "emqx.acl_file", [ + {datatype, string}, + hidden +]}. + +%% @doc Enable ACL cache for publish. +{mapping, "enable_acl_cache", "emqx.enable_acl_cache", [ + {default, on}, + {datatype, flag} +]}. + +%% @doc ACL cache time-to-live. +{mapping, "acl_cache_ttl", "emqx.acl_cache_ttl", [ + {default, "1m"}, + {datatype, {duration, ms}} +]}. + +%% @doc ACL cache size. +{mapping, "acl_cache_max_size", "emqx.acl_cache_max_size", [ + {default, 32}, + {datatype, integer}, + {validators, ["range:gt_0"]} +]}. + +%% @doc Action when acl check reject current operation +{mapping, "acl_deny_action", "emqx.acl_deny_action", [ + {default, ignore}, + {datatype, {enum, [ignore, disconnect]}} +]}. + +{validator, "range:gt_0", "must greater than 0", + fun(X) -> X > 0 end +}. + +%%-------------------------------------------------------------------- +%% MQTT Protocol +%%-------------------------------------------------------------------- + +%% @doc Response Topic Prefix +{mapping, "mqtt.response_topic_prefix", "emqx.response_topic_prefix",[ + {datatype, string} +]}. + +%% @doc Max Packet Size Allowed, 1MB by default. +{mapping, "mqtt.max_packet_size", "emqx.max_packet_size", [ + {default, "1MB"}, + {datatype, bytesize} +]}. + +%% @doc Set the Max ClientId Length Allowed. +{mapping, "mqtt.max_clientid_len", "emqx.max_clientid_len", [ + {default, 65535}, + {datatype, integer} +]}. + +%% @doc Set the Maximum topic levels. +{mapping, "mqtt.max_topic_levels", "emqx.max_topic_levels", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Set the Maximum QoS allowed. +{mapping, "mqtt.max_qos_allowed", "emqx.max_qos_allowed", [ + {default, 2}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +%% @doc Set the Maximum topic alias. +{mapping, "mqtt.max_topic_alias", "emqx.max_topic_alias", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Whether the server supports MQTT retained messages. +{mapping, "mqtt.retain_available", "emqx.mqtt_retain_available", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports MQTT Wildcard Subscriptions. +{mapping, "mqtt.wildcard_subscription", "emqx.mqtt_wildcard_subscription", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports MQTT Shared Subscriptions. +{mapping, "mqtt.shared_subscription", "emqx.mqtt_shared_subscription", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether to ignore loop delivery of messages.(for mqtt v3.1.1) +{mapping, "mqtt.ignore_loop_deliver", "emqx.mqtt_ignore_loop_deliver", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%%-------------------------------------------------------------------- +%% Zones +%%-------------------------------------------------------------------- + +%% @doc Idle timeout of the MQTT connection. +{mapping, "zone.$name.idle_timeout", "emqx.zones", [ + {default, "15s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "zone.$name.allow_anonymous", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "zone.$name.acl_nomatch", "emqx.zones", [ + {datatype, {enum, [allow, deny]}} +]}. + +%% @doc Enable ACL check. +{mapping, "zone.$name.enable_acl", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Action when acl check reject current operation +{mapping, "zone.$name.acl_deny_action", "emqx.zones", [ + {default, ignore}, + {datatype, {enum, [ignore, disconnect]}} +]}. + +%% @doc Enable Ban. +{mapping, "zone.$name.enable_ban", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Enable per connection statistics. +{mapping, "zone.$name.enable_stats", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Publish limit of the MQTT connections. +{mapping, "zone.$name.publish_limit", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Max Packet Size Allowed, 64K by default. +{mapping, "zone.$name.max_packet_size", "emqx.zones", [ + {datatype, bytesize} +]}. + +%% @doc Set the Max ClientId Length Allowed. +{mapping, "zone.$name.max_clientid_len", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Set the Maximum topic levels. +{mapping, "zone.$name.max_topic_levels", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Set the Maximum QoS allowed. +{mapping, "zone.$name.max_qos_allowed", "emqx.zones", [ + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +%% @doc Set the Maximum topic alias. +{mapping, "zone.$name.max_topic_alias", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Whether the server supports retained messages. +{mapping, "zone.$name.retain_available", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports Wildcard Subscriptions. +{mapping, "zone.$name.wildcard_subscription", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports Shared Subscriptions. +{mapping, "zone.$name.shared_subscription", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Server Keepalive +{mapping, "zone.$name.server_keepalive", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Keepalive backoff +{mapping, "zone.$name.keepalive_backoff", "emqx.zones", [ + {default, 0.75}, + {datatype, float} +]}. + +%% @doc Max Number of Subscriptions Allowed. +{mapping, "zone.$name.max_subscriptions", "emqx.zones", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Upgrade QoS according to subscription? +{mapping, "zone.$name.upgrade_qos", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Max number of QoS 1 and 2 messages that can be “inflight” at one time. +%% 0 means no limit +{mapping, "zone.$name.max_inflight", "emqx.zones", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Retry interval for redelivering QoS1/2 messages. +{mapping, "zone.$name.retry_interval", "emqx.zones", [ + {default, "20s"}, + {datatype, {duration, ms}} +]}. + +%% @doc Max Packets that Awaiting PUBREL, 0 means no limit +{mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Awaiting PUBREL timeout +{mapping, "zone.$name.await_rel_timeout", "emqx.zones", [ + {default, "300s"}, + {datatype, {duration, ms}} +]}. + +%% @doc Ignore loop delivery of messages +{mapping, "zone.$name.ignore_loop_deliver", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Session Expiry Interval +{mapping, "zone.$name.session_expiry_interval", "emqx.zones", [ + {default, "2h"}, + {datatype, {duration, s}} +]}. + +%% @doc Max queue length. Enqueued messages when persistent client +%% disconnected, or inflight window is full. 0 means no limit. +{mapping, "zone.$name.max_mqueue_len", "emqx.zones", [ + {default, 1000}, + {datatype, integer} +]}. + +%% @doc Topic Priorities, comma separated topic=priority pairs, +%% where priority should be integer in range 1-255 (inclusive) +%% 1 being the lowest and 255 being the highest. +%% default value `none` to indicate no priority table, hence all +%% messages are treated equal, which means either highest ('infinity'), +%% or lowest (0) depending on mqueue_default_priority config. +{mapping, "zone.$name.mqueue_priorities", "emqx.zones", [ + {default, "none"}, + {datatype, string} +]}. + +%% @doc Default priority for topics not in priority table. +{mapping, "zone.$name.mqueue_default_priority", "emqx.zones", [ + {default, lowest}, + {datatype, {enum, [highest, lowest]}} +]}. + +%% @doc Queue Qos0 messages? +{mapping, "zone.$name.mqueue_store_qos0", "emqx.zones", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Force connection/session process GC after this number of +%% messages | bytes passed through. +%% Numbers delimited by `|'. Zero or negative is to disable. +{mapping, "zone.$name.force_gc_policy", "emqx.zones", [ + {default, "0 | 0MB"}, + {datatype, string} + ]}. + +%% @doc Max message queue length and total heap size to force shutdown +%% connection/session process. +%% Message queue here is the Erlang process mailbox, but not the number +%% of queued MQTT messages of QoS 1 and 2. +%% Zero or negative is to disable. +{mapping, "zone.$name.force_shutdown_policy", "emqx.zones", [ + {default, "0 | 0MB"}, + {datatype, string} +]}. + +{mapping, "zone.$name.mountpoint", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Use username replace client id +{mapping, "zone.$name.use_username_as_clientid", "emqx.zones", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{translation, "emqx.zones", fun(Conf) -> + Mapping = fun("retain_available", Val) -> + {mqtt_retain_available, Val}; + ("wildcard_subscription", Val) -> + {mqtt_wildcard_subscription, Val}; + ("shared_subscription", Val) -> + {mqtt_shared_subscription, Val}; + ("publish_limit", Val) -> + [Limit, Duration] = string:tokens(Val, ", "), + PubLimit = case cuttlefish_duration:parse(Duration, s) of + Secs when is_integer(Secs) -> + {list_to_integer(Limit) / Secs, list_to_integer(Limit)}; + {error, Reason} -> + error(Reason) + end, + {publish_limit, PubLimit}; + ("force_gc_policy", Val) -> + [Count, Bytes] = string:tokens(Val, "| "), + GcPolicy = case cuttlefish_bytesize:parse(Bytes) of + {error, Reason} -> + error(Reason); + Bytes1 -> + #{bytes => Bytes1, + count => list_to_integer(Count)} + end, + {force_gc_policy, GcPolicy}; + ("force_shutdown_policy", Val) -> + [Len, Siz] = string:tokens(Val, "| "), + ShutdownPolicy = case cuttlefish_bytesize:parse(Siz) of + {error, Reason} -> + error(Reason); + Siz1 -> + #{message_queue_len => list_to_integer(Len), + max_heap_size => Siz1} + end, + {force_shutdown_policy, ShutdownPolicy}; + ("mqueue_priorities", Val) -> + case Val of + "none" -> none; % NO_PRIORITY_TABLE + _ -> + lists:foldl(fun(T, Acc) -> + %% NOTE: space in "= " is intended + [{Topic, Prio}] = string:tokens(T, "= "), + P = list_to_integer(Prio), + (P < 0 orelse P > 255) andalso error({bad_priority, Topic, Prio}), + maps:put(iolist_to_binary(Topic), P, Acc) + end, string:tokens(Val, ",")) + end; + (Opt, Val) -> + {list_to_atom(Opt), Val} + end, + maps:to_list( + lists:foldl( + fun({["zone", Name, Opt], Val}, Zones) -> + maps:update_with(list_to_atom(Name), + fun(Opts) -> [Mapping(Opt, Val)|Opts] end, + [Mapping(Opt, Val)], Zones) + end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("zone.", Conf)))) +end}. + +%%-------------------------------------------------------------------- +%% Listeners +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% TCP Listeners + +{mapping, "listener.tcp.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.tcp.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.mountpoint", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.tcp.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +{mapping, "listener.tcp.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt]}} +]}. + +{mapping, "listener.tcp.$name.backlog", "emqx.listeners", [ + {datatype, integer}, + {default, 1024} +]}. + +{mapping, "listener.tcp.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.tcp.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.tcp.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.tcp.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.tcp.$name.reuseaddr", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +%%-------------------------------------------------------------------- +%% SSL Listeners + +{mapping, "listener.ssl.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.ssl.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.mountpoint", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ssl.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.ssl.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.ssl.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.ssl.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ssl.$name.reuseaddr", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ssl.$name.tls_versions", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.handshake_timeout", "emqx.listeners", [ + {default, "15s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ssl.$name.dhfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.keyfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.certfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.cacertfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.verify", "emqx.listeners", [ + {datatype, atom} +]}. + +{mapping, "listener.ssl.$name.fail_if_no_peer_cert", "emqx.listeners", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.ssl.$name.secure_renegotiate", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.reuse_sessions", "emqx.listeners", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.honor_cipher_order", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt]}} +]}. + +%%-------------------------------------------------------------------- +%% MQTT/WebSocket Listeners + +{mapping, "listener.ws.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.ws.$name.mqtt_path", "emqx.listeners", [ + {default, "/mqtt"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ws.$name.mountpoint", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ws.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ws.$name.verify_protocol_header", "emqx.listeners", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "listener.ws.$name.proxy_address_header", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + +{mapping, "listener.ws.$name.proxy_port_header", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + +{mapping, "listener.ws.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ws.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ws.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.ws.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.ws.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.ws.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +%%-------------------------------------------------------------------- +%% MQTT/WebSocket/SSL Listeners + +{mapping, "listener.wss.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.wss.$name.mqtt_path", "emqx.listeners", [ + {default, "/mqtt"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.mountpoint", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.rate_limit", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.verify_protocol_header", "emqx.listeners", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.proxy_address_header", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + +{mapping, "listener.wss.$name.proxy_port_header", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + +{mapping, "listener.wss.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +%%{mapping, "listener.wss.$name.handshake_timeout", "emqx.listeners", [ +%% {default, "15s"}, +%% {datatype, {duration, ms}} +%%]}. + +{mapping, "listener.wss.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.wss.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.wss.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.wss.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.wss.$name.tls_versions", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.keyfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.certfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.cacertfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.verify", "emqx.listeners", [ + {datatype, atom} +]}. + +{mapping, "listener.wss.$name.fail_if_no_peer_cert", "emqx.listeners", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.wss.$name.secure_renegotiate", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.reuse_sessions", "emqx.listeners", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.honor_cipher_order", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt]}} +]}. + +{translation, "emqx.listeners", fun(Conf) -> + + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + + Atom = fun(undefined) -> undefined; (S) -> list_to_atom(S) end, + + Access = fun(S) -> + [A, CIDR] = string:tokens(S, " "), + {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} + end, + + AccOpts = fun(Prefix) -> + case cuttlefish_variable:filter_by_prefix(Prefix ++ ".access", Conf) of + [] -> []; + Rules -> [{access_rules, [Access(Rule) || {_, Rule} <- Rules]}] + end + end, + + MountPoint = fun(undefined) -> undefined; (S) -> list_to_binary(S) end, + + Ratelimit = fun(undefined) -> + undefined; + (S) -> + list_to_tuple([list_to_integer(Token) || Token <- string:tokens(S, ",")]) + end, + + LisOpts = fun(Prefix) -> + Filter([{acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, + {mqtt_path, cuttlefish:conf_get(Prefix ++ ".mqtt_path", Conf, undefined)}, + {max_connections, cuttlefish:conf_get(Prefix ++ ".max_connections", Conf)}, + {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)}, + {active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)}, + {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)}, + {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))}, + {rate_limit, Ratelimit(cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined))}, + {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)}, + {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)}, + {mountpoint, MountPoint(cuttlefish:conf_get(Prefix ++ ".mountpoint", Conf, undefined))}, + {verify_protocol_header, cuttlefish:conf_get(Prefix ++ ".verify_protocol_header", Conf, undefined)}, + {peer_cert_as_username, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_username", Conf, undefined)}, + {proxy_port_header, cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, undefined)}, + {proxy_address_header, cuttlefish:conf_get(Prefix ++ ".proxy_address_header", Conf, undefined)} | AccOpts(Prefix)]) + end, + TcpOpts = fun(Prefix) -> + Filter([{backlog, cuttlefish:conf_get(Prefix ++ ".backlog", Conf, undefined)}, + {send_timeout, cuttlefish:conf_get(Prefix ++ ".send_timeout", Conf, undefined)}, + {send_timeout_close, cuttlefish:conf_get(Prefix ++ ".send_timeout_close", Conf, undefined)}, + {recbuf, cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)}, + {sndbuf, cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)}, + {buffer, cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)}, + {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)}, + {reuseaddr, cuttlefish:conf_get(Prefix ++ ".reuseaddr", Conf, undefined)}]) + end, + + SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, + + SslOpts = fun(Prefix) -> + Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of + undefined -> undefined; + L -> [list_to_atom(V) || V <- L] + end, + Filter([{versions, Versions}, + {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))}, + {handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf, undefined)}, + {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, + {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, + {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, + {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, + {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, + {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, + {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, + {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, + {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) + end, + + TcpListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + case cuttlefish:conf_get(Prefix, Conf, undefined) of + undefined -> []; + ListenOn -> + [{Atom(Type), ListenOn, [{tcp_options, TcpOpts(Prefix)} | LisOpts(Prefix)]}] + end + end, + + SslListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + case cuttlefish:conf_get(Prefix, Conf, undefined) of + undefined -> + []; + ListenOn -> + [{Atom(Type), ListenOn, [{tcp_options, TcpOpts(Prefix)}, + {ssl_options, SslOpts(Prefix)} | LisOpts(Prefix)]}] + end + end, + + lists:flatten([TcpListeners(Type, Name) || {["listener", Type, Name], ListenOn} + <- cuttlefish_variable:filter_by_prefix("listener.tcp", Conf) + ++ cuttlefish_variable:filter_by_prefix("listener.ws", Conf)] + ++ + [SslListeners(Type, Name) || {["listener", Type, Name], ListenOn} + <- cuttlefish_variable:filter_by_prefix("listener.ssl", Conf) + ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)]) +end}. + +%%-------------------------------------------------------------------- +%% Bridges +%%-------------------------------------------------------------------- +{mapping, "bridge.$name.mqueue_type", "emqx.bridges", [ + {datatype, {enum, [memory, disk]}} +]}. + +{mapping, "bridge.$name.address", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.proto_ver", "emqx.bridges", [ + {datatype, {enum, [mqttv3, mqttv4, mqttv5]}} +]}. + +{mapping, "bridge.$name.client_id", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.clean_start", "emqx.bridges", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "bridge.$name.username", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.password", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.mountpoint", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.forwards", "emqx.bridges", [ + {datatype, string}, + {default, ""} +]}. + +{mapping, "bridge.$name.ssl", "emqx.bridges", [ + {datatype, flag}, + {default, off} +]}. + +{mapping, "bridge.$name.cacertfile", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.certfile", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.keyfile", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.ciphers", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.max_pending_messages", "emqx.bridges", [ + {default, 10000}, + {datatype, integer} +]}. + +{mapping, "bridge.$name.keepalive", "emqx.bridges", [ + {default, "10s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "bridge.$name.tls_versions", "emqx.bridges", [ + {datatype, string}, + {default, "tlsv1,tlsv1.1,tlsv1.2"} +]}. + +{mapping, "bridge.$name.subscription.$id.topic", "emqx.bridges", [ + {datatype, string} +]}. + +{mapping, "bridge.$name.subscription.$id.qos", "emqx.bridges", [ + {datatype, integer} +]}. + +{mapping, "bridge.$name.start_type", "emqx.bridges", [ + {datatype, {enum, [manual, auto]}}, + {default, auto} +]}. + +{mapping, "bridge.$name.reconnect_interval", "emqx.bridges", [ + {default, "30s"}, + {datatype, {duration, ms}} +]}. + + +{translation, "emqx.bridges", fun(Conf) -> + Split = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, + + IsSsl = fun(cacertfile) -> true; + (certfile) -> true; + (keyfile) -> true; + (ciphers) -> true; + (tls_versions) -> true; + (_Opt) -> false + end, + + Parse = fun(tls_versions, Vers) -> + {versions, [list_to_atom(S) || S <- Split(Vers)]}; + (ciphers, Ciphers) -> + {ciphers, Split(Ciphers)}; + (Opt, Val) -> + {Opt, Val} + end, + + Merge = fun(forwards, Val, Opts) -> + [{forwards, string:tokens(Val, ",")}|Opts]; + (Opt, Val, Opts) -> + case IsSsl(Opt) of + true -> + SslOpts = [Parse(Opt, Val)|proplists:get_value(ssl_opts, Opts, [])], + lists:ukeymerge(1, [{ssl_opts, SslOpts}], lists:usort(Opts)); + false -> + [{Opt, Val}|Opts] + end + end, + + Subscriptions = fun(Name) -> + Configs = cuttlefish_variable:filter_by_prefix("bridge." ++ Name ++ ".subscription", Conf), + lists:zip([Topic || {_, Topic} <- lists:sort([{I, Topic} || {[_, _, "subscription", I, "topic"], Topic} <- Configs])], + [QoS || {_, QoS} <- lists:sort([{I, QoS} || {[_, _, "subscription", I, "qos"], QoS} <- Configs])]) + end, + + maps:to_list( + lists:foldl( + fun({["bridge", Name, Opt], Val}, Acc) -> + %% e.g #{aws => [{OptKey, OptVal}]} + Init = [{list_to_atom(Opt), Val},{subscriptions, Subscriptions(Name)}], + maps:update_with(list_to_atom(Name), + fun(Opts) -> Merge(list_to_atom(Opt), Val, Opts) end, Init, Acc); + (_, Acc) -> Acc + end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("bridge.", Conf)))) + +end}. + +%%-------------------------------------------------------------------- +%% Modules +%%-------------------------------------------------------------------- + +{mapping, "module.presence", "emqx.modules", [ + {default, off}, + {datatype, flag} +]}. + +{mapping, "module.presence.qos", "emqx.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription", "emqx.modules", [ + {default, off}, + {datatype, flag} +]}. + +{mapping, "module.subscription.$id.topic", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.subscription.$id.qos", "emqx.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.rewrite", "emqx.modules", [ + {default, off}, + {datatype, flag} +]}. + +{mapping, "module.rewrite.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{translation, "emqx.modules", fun(Conf) -> + Subscriptions = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), + QosList = [Qos || {_, Qos} <- lists:sort([{I, Qos} || {[_,"subscription", I,"qos"], Qos} <- List])], + TopicList = [iolist_to_binary(Topic) || {_, Topic} <- + lists:sort([{I, Topic} || {[_,"subscription", I, "topic"], Topic} <- List])], + lists:zip(TopicList, QosList) + end, + Rewrites = fun() -> + Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf), + lists:map(fun({[_, "rewrite", "rule", I], Rule}) -> + [Topic, Re, Dest] = string:tokens(Rule, " "), + {rewrite, list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} + end, Rules) + end, + lists:append([ + case cuttlefish:conf_get("module.presence", Conf) of %% Presence + true -> [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}]; + false -> [] + end, + case cuttlefish:conf_get("module.subscription", Conf) of %% Subscription + true -> [{emqx_mod_subscription, Subscriptions()}]; + false -> [] + end, + case cuttlefish:conf_get("module.rewrite", Conf) of %% Rewrite + true -> [{emqx_mod_rewrite, Rewrites()}]; + false -> [] + end + ]) +end}. + +%%------------------------------------------------------------------- +%% Plugins +%%------------------------------------------------------------------- + +{mapping, "plugins.etc_dir", "emqx.plugins_etc_dir", [ + {datatype, string} +]}. + +{mapping, "plugins.loaded_file", "emqx.plugins_loaded_file", [ + {datatype, string} +]}. + +{mapping, "plugins.expand_plugins_dir", "emqx.expand_plugins_dir", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Broker +%%-------------------------------------------------------------------- + +{mapping, "broker.sys_interval", "emqx.broker_sys_interval", [ + {datatype, {duration, ms}}, + {default, "1m"} +]}. + +{mapping, "broker.enable_session_registry", "emqx.enable_session_registry", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "broker.session_locking_strategy", "emqx.session_locking_strategy", [ + {default, quorum}, + {datatype, {enum, [local,one,quorum,all]}} +]}. + +%% @doc Shared Subscription Dispatch Strategy. +{mapping, "broker.shared_subscription_strategy", "emqx.shared_subscription_strategy", [ + {default, round_robbin}, + {datatype, + {enum, + [random, %% randomly pick a subscriber + round_robbin, %% round robin alive subscribers one message after another + sticky, %% pick a random subscriber and stick to it + hash %% hash client ID to a group member + ]}} +]}. + +%% @doc Enable or disable shared dispatch acknowledgement for QoS1 and QoS2 messages +{mapping, "broker.shared_dispatch_ack_enabled", "emqx.shared_dispatch_ack_enabled", + [ {default, false}, + {datatype, {enum, [true, false]}} + ]}. + +{mapping, "broker.route_batch_clean", "emqx.route_batch_clean", [ + {default, on}, + {datatype, flag} +]}. + +%%-------------------------------------------------------------------- +%% System Monitor +%%-------------------------------------------------------------------- + +%% @doc Long GC, don't monitor in production mode for: +%% https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 +{mapping, "sysmon.long_gc", "emqx.sysmon", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Long Schedule(ms) +{mapping, "sysmon.long_schedule", "emqx.sysmon", [ + {default, 1000}, + {datatype, integer} +]}. + +%% @doc Large Heap +{mapping, "sysmon.large_heap", "emqx.sysmon", [ + {default, "8MB"}, + {datatype, bytesize} +]}. + +%% @doc Monitor Busy Port +{mapping, "sysmon.busy_port", "emqx.sysmon", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Monitor Busy Dist Port +{mapping, "sysmon.busy_dist_port", "emqx.sysmon", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{translation, "emqx.sysmon", fun(Conf) -> + [{long_gc, cuttlefish:conf_get("sysmon.long_gc", Conf)}, + {long_schedule, cuttlefish:conf_get("sysmon.long_schedule", Conf)}, + {large_heap, cuttlefish:conf_get("sysmon.large_heap", Conf)}, + {busy_port, cuttlefish:conf_get("sysmon.busy_port", Conf)}, + {busy_dist_port, cuttlefish:conf_get("sysmon.busy_dist_port", Conf)}] +end}. diff --git a/rebar.config b/rebar.config new file mode 100644 index 000000000..d3dabae81 --- /dev/null +++ b/rebar.config @@ -0,0 +1,29 @@ +{deps, [{jsx, "2.9.0"}, + {gproc, "0.8.0"}, + {cowboy, "2.4.0"} + ]}. + +%% appended to deps in rebar.config.script +{github_emqx_deps, + [{gen_rpc, "2.3.0"}, + {ekka, "v0.5.1"}, + {esockd, "v5.4.3"}, + {cuttlefish, "v2.2.0"} + ]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {d, 'APPLICATION', emqx}]}. +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{plugins, [coveralls]}. + diff --git a/rebar.config.script b/rebar.config.script new file mode 100644 index 000000000..aed271af8 --- /dev/null +++ b/rebar.config.script @@ -0,0 +1,26 @@ + +CONFIG0 = case os:getenv("REBAR_GIT_CLONE_OPTIONS") of + "--depth 1" -> + CONFIG; + _ -> + os:putenv("REBAR_GIT_CLONE_OPTIONS", "--depth 1"), + CONFIG + end, + +CONFIG1 = case os:getenv("TRAVIS") of + "true" -> + JobId = os:getenv("TRAVIS_JOB_ID"), + [{coveralls_service_job_id, JobId}, + {coveralls_coverdata, "_build/test/cover/*.coverdata"}, + {coveralls_service_name , "travis-ci"} | CONFIG]; + _ -> + CONFIG + end, + +{_, Deps} = lists:keyfind(deps, 1, CONFIG1), +{_, OurDeps} = lists:keyfind(github_emqx_deps, 1, CONFIG1), +UrlPrefix = "https://github.com/emqx/", +NewDeps = Deps ++ [{Name, {git, UrlPrefix ++ atom_to_list(Name), {branch, Branch}}} || {Name, Branch} <- OurDeps], +CONFIG2 = lists:keystore(deps, 1, CONFIG1, {deps, NewDeps}), + +CONFIG2. diff --git a/rebar.lock b/rebar.lock deleted file mode 100644 index 1364f95ae..000000000 --- a/rebar.lock +++ /dev/null @@ -1,36 +0,0 @@ -[{<<"esockd">>, - {git,"https://github.com/emqtt/esockd", - {ref,"87d0d3b672e0f25e474f5f8298da568cbb6b168a"}}, - 0}, - {<<"gen_logger">>, - {git,"https://github.com/emqtt/gen_logger.git", - {ref,"f6e9f2f373d99f41ffe0579ab5a5f3b19472c9c5"}}, - 1}, - {<<"goldrush">>, - {git,"https://github.com/basho/goldrush.git", - {ref,"8f1b715d36b650ec1e1f5612c00e28af6ab0de82"}}, - 1}, - {<<"gproc">>, - {git,"https://github.com/uwiger/gproc", - {ref,"01c8fbfdd5e4701e8e4b57b0c8279872f9574b0b"}}, - 0}, - {<<"lager">>, - {git,"https://github.com/basho/lager", - {ref,"81eaef0ce98fdbf64ab95665e3bc2ec4b24c7dac"}}, - 0}, - {<<"lager_syslog">>, - {git,"https://github.com/basho/lager_syslog", - {ref,"126dd0284fcac9b01613189a82facf8d803411a2"}}, - 0}, - {<<"mochiweb">>, - {git,"https://github.com/emqtt/mochiweb", - {ref,"c75d88e451b4fe26580a58223f645d99482f51af"}}, - 0}, - {<<"pbkdf2">>, - {git,"https://github.com/comtihon/erlang-pbkdf2.git", - {ref,"7076584f5377e98600a7e2cb81980b2992fb2f71"}}, - 0}, - {<<"syslog">>, - {git,"git://github.com/Vagabond/erlang-syslog", - {ref,"0e4f0e95c361af298c5d1d17ceccfa831efc036d"}}, - 1}]. diff --git a/src/emqttd.app.src b/src/emqttd.app.src deleted file mode 100644 index bd42f9b7d..000000000 --- a/src/emqttd.app.src +++ /dev/null @@ -1,12 +0,0 @@ -{application,emqttd, - [{description,"Erlang MQTT Broker"}, - {vsn,"2.3.11"}, - {modules,[]}, - {registered,[emqttd_sup]}, - {applications,[kernel,stdlib,gproc,lager,esockd,mochiweb, - lager_syslog,pbkdf2,bcrypt]}, - {env,[]}, - {mod,{emqttd_app,[]}}, - {maintainers,["Feng Lee "]}, - {licenses,["Apache-2.0"]}, - {links,[{"Github","https://github.com/emqtt/emqttd"}]}]}. diff --git a/src/emqttd.erl b/src/emqttd.erl deleted file mode 100644 index ecea2ca18..000000000 --- a/src/emqttd.erl +++ /dev/null @@ -1,181 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc EMQ Main Module. - --module(emqttd). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --export([start/0, env/1, env/2, is_running/1, stop/0]). - -%% PubSub API --export([subscribe/1, subscribe/2, subscribe/3, publish/1, - unsubscribe/1, unsubscribe/2]). - -%% PubSub Management API --export([setqos/3, topics/0, subscriptions/1, subscribers/1, subscribed/2]). - -%% Hooks API --export([hook/4, hook/3, unhook/2, run_hooks/2, run_hooks/3]). - -%% Debug API --export([dump/0]). - -%% Shutdown and reboot --export([shutdown/0, shutdown/1, reboot/0]). - --type(subid() :: binary()). - --type(subscriber() :: pid() | subid() | {subid(), pid()}). - --type(suboption() :: local | {qos, non_neg_integer()} | {share, {'$queue' | binary()}}). - --export_type([subscriber/0, suboption/0]). - --define(APP, ?MODULE). - -%%-------------------------------------------------------------------- -%% Bootstrap, environment, configuration, is_running... -%%-------------------------------------------------------------------- - -%% @doc Start emqttd application. --spec(start() -> ok | {error, term()}). -start() -> application:start(?APP). - -%% @doc Stop emqttd application. --spec(stop() -> ok | {error, term()}). -stop() -> application:stop(?APP). - -%% @doc Environment --spec(env(Key :: atom()) -> {ok, any()} | undefined). -env(Key) -> application:get_env(?APP, Key). - -%% @doc Get environment --spec(env(Key :: atom(), Default :: any()) -> undefined | any()). -env(Key, Default) -> application:get_env(?APP, Key, Default). - -%% @doc Is running? --spec(is_running(node()) -> boolean()). -is_running(Node) -> - case rpc:call(Node, erlang, whereis, [?APP]) of - {badrpc, _} -> false; - undefined -> false; - Pid when is_pid(Pid) -> true - end. - -%%-------------------------------------------------------------------- -%% PubSub APIs -%%-------------------------------------------------------------------- - -%% @doc Subscribe --spec(subscribe(iodata()) -> ok | {error, term()}). -subscribe(Topic) -> - emqttd_server:subscribe(iolist_to_binary(Topic)). - --spec(subscribe(iodata(), subscriber()) -> ok | {error, term()}). -subscribe(Topic, Subscriber) -> - emqttd_server:subscribe(iolist_to_binary(Topic), Subscriber). - --spec(subscribe(iodata(), subscriber(), [suboption()]) -> ok | {error, term()}). -subscribe(Topic, Subscriber, Options) -> - emqttd_server:subscribe(iolist_to_binary(Topic), Subscriber, Options). - -%% @doc Publish MQTT Message --spec(publish(mqtt_message()) -> {ok, mqtt_delivery()} | ignore). -publish(Msg) -> - emqttd_server:publish(Msg). - -%% @doc Unsubscribe --spec(unsubscribe(iodata()) -> ok | {error, term()}). -unsubscribe(Topic) -> - emqttd_server:unsubscribe(iolist_to_binary(Topic)). - --spec(unsubscribe(iodata(), subscriber()) -> ok | {error, term()}). -unsubscribe(Topic, Subscriber) -> - emqttd_server:unsubscribe(iolist_to_binary(Topic), Subscriber). - --spec(setqos(binary(), subscriber(), mqtt_qos()) -> ok). -setqos(Topic, Subscriber, Qos) -> - emqttd_server:setqos(iolist_to_binary(Topic), Subscriber, Qos). - --spec(topics() -> [binary()]). -topics() -> emqttd_router:topics(). - --spec(subscribers(iodata()) -> list(subscriber())). -subscribers(Topic) -> - emqttd_server:subscribers(iolist_to_binary(Topic)). - --spec(subscriptions(subscriber()) -> [{emqttd:subscriber(), binary(), list(emqttd:suboption())}]). -subscriptions(Subscriber) -> - emqttd_server:subscriptions(Subscriber). - --spec(subscribed(iodata(), subscriber()) -> boolean()). -subscribed(Topic, Subscriber) -> - emqttd_server:subscribed(iolist_to_binary(Topic), Subscriber). - -%%-------------------------------------------------------------------- -%% Hooks API -%%-------------------------------------------------------------------- - --spec(hook(atom(), function() | {emqttd_hooks:hooktag(), function()}, list(any())) - -> ok | {error, term()}). -hook(Hook, TagFunction, InitArgs) -> - emqttd_hooks:add(Hook, TagFunction, InitArgs). - --spec(hook(atom(), function() | {emqttd_hooks:hooktag(), function()}, list(any()), integer()) - -> ok | {error, term()}). -hook(Hook, TagFunction, InitArgs, Priority) -> - emqttd_hooks:add(Hook, TagFunction, InitArgs, Priority). - --spec(unhook(atom(), function() | {emqttd_hooks:hooktag(), function()}) - -> ok | {error, term()}). -unhook(Hook, TagFunction) -> - emqttd_hooks:delete(Hook, TagFunction). - --spec(run_hooks(atom(), list(any())) -> ok | stop). -run_hooks(Hook, Args) -> - emqttd_hooks:run(Hook, Args). - --spec(run_hooks(atom(), list(any()), any()) -> {ok | stop, any()}). -run_hooks(Hook, Args, Acc) -> - emqttd_hooks:run(Hook, Args, Acc). - -%%-------------------------------------------------------------------- -%% Shutdown and reboot -%%-------------------------------------------------------------------- - -shutdown() -> - shutdown(normal). - -shutdown(Reason) -> - lager:error("EMQ shutdown for ~s", [Reason]), - emqttd_plugins:unload(), - lists:foreach(fun application:stop/1, [emqttd, ekka, mochiweb, esockd, gproc]). - -reboot() -> - lists:foreach(fun application:start/1, [gproc, esockd, mochiweb, ekka, emqttd]). - -%%-------------------------------------------------------------------- -%% Debug -%%-------------------------------------------------------------------- - -dump() -> lists:append([emqttd_server:dump(), emqttd_router:dump()]). - diff --git a/src/emqttd_access_control.erl b/src/emqttd_access_control.erl deleted file mode 100644 index 0b74e2dc2..000000000 --- a/src/emqttd_access_control.erl +++ /dev/null @@ -1,180 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_access_control). - --behaviour(gen_server). - --author("Feng Lee "). - --include("emqttd.hrl"). - -%% API Function Exports --export([start_link/0, auth/2, check_acl/3, reload_acl/0, lookup_mods/1, - register_mod/3, register_mod/4, unregister_mod/2, stop/0]). - -%% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --define(SERVER, ?MODULE). - --define(ACCESS_CONTROL_TAB, mqtt_access_control). - --type(password() :: undefined | binary()). - --record(state, {}). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -%% @doc Start access control server. --spec(start_link() -> {ok, pid()} | ignore | {error, term()}). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -%% @doc Authenticate MQTT Client. --spec(auth(Client :: mqtt_client(), Password :: password()) -> ok | {ok, boolean()} | {error, term()}). -auth(Client, Password) when is_record(Client, mqtt_client) -> - auth(Client, Password, lookup_mods(auth)). -auth(_Client, _Password, []) -> - case emqttd:env(allow_anonymous, false) of - true -> ok; - false -> {error, "No auth module to check!"} - end; -auth(Client, Password, [{Mod, State, _Seq} | Mods]) -> - case catch Mod:check(Client, Password, State) of - ok -> ok; - {ok, IsSuper} -> {ok, IsSuper}; - ignore -> auth(Client, Password, Mods); - {error, Reason} -> {error, Reason}; - {'EXIT', Error} -> {error, Error} - end. - -%% @doc Check ACL --spec(check_acl(Client, PubSub, Topic) -> allow | deny when - Client :: mqtt_client(), - PubSub :: pubsub(), - Topic :: binary()). -check_acl(Client, PubSub, Topic) when ?PS(PubSub) -> - check_acl(Client, PubSub, Topic, lookup_mods(acl)). - -check_acl(_Client, _PubSub, _Topic, []) -> - emqttd:env(acl_nomatch, allow); -check_acl(Client, PubSub, Topic, [{Mod, State, _Seq}|AclMods]) -> - case Mod:check_acl({Client, PubSub, Topic}, State) of - allow -> allow; - deny -> deny; - ignore -> check_acl(Client, PubSub, Topic, AclMods) - end. - -%% @doc Reload ACL Rules. --spec(reload_acl() -> list(ok | {error, already_existed})). -reload_acl() -> - [Mod:reload_acl(State) || {Mod, State, _Seq} <- lookup_mods(acl)]. - -%% @doc Register Authentication or ACL module. --spec(register_mod(auth | acl, atom(), list()) -> ok | {error, term()}). -register_mod(Type, Mod, Opts) when Type =:= auth; Type =:= acl-> - register_mod(Type, Mod, Opts, 0). - --spec(register_mod(auth | acl, atom(), list(), non_neg_integer()) -> ok | {error, term()}). -register_mod(Type, Mod, Opts, Seq) when Type =:= auth; Type =:= acl-> - gen_server:call(?SERVER, {register_mod, Type, Mod, Opts, Seq}). - -%% @doc Unregister authentication or ACL module --spec(unregister_mod(Type :: auth | acl, Mod :: atom()) -> ok | {error, not_found | term()}). -unregister_mod(Type, Mod) when Type =:= auth; Type =:= acl -> - gen_server:call(?SERVER, {unregister_mod, Type, Mod}). - -%% @doc Lookup authentication or ACL modules. --spec(lookup_mods(auth | acl) -> list()). -lookup_mods(Type) -> - case ets:lookup(?ACCESS_CONTROL_TAB, tab_key(Type)) of - [] -> []; - [{_, Mods}] -> Mods - end. - -tab_key(auth) -> auth_modules; -tab_key(acl) -> acl_modules. - -%% @doc Stop access control server. -stop() -> gen_server:call(?MODULE, stop). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([]) -> - ets:new(?ACCESS_CONTROL_TAB, [set, named_table, protected, {read_concurrency, true}]), - {ok, #state{}}. - -handle_call({register_mod, Type, Mod, Opts, Seq}, _From, State) -> - Mods = lookup_mods(Type), - Existed = lists:keyfind(Mod, 1, Mods), - {reply, if_existed(Existed, fun() -> - case catch Mod:init(Opts) of - {ok, ModState} -> - NewMods = lists:sort(fun({_, _, Seq1}, {_, _, Seq2}) -> - Seq1 >= Seq2 - end, [{Mod, ModState, Seq} | Mods]), - ets:insert(?ACCESS_CONTROL_TAB, {tab_key(Type), NewMods}), - ok; - {error, Error} -> - {error, Error}; - {'EXIT', Reason} -> - {error, Reason} - end - end), State}; - -handle_call({unregister_mod, Type, Mod}, _From, State) -> - Mods = lookup_mods(Type), - case lists:keyfind(Mod, 1, Mods) of - false -> - {reply, {error, not_found}, State}; - _ -> - ets:insert(?ACCESS_CONTROL_TAB, {tab_key(Type), lists:keydelete(Mod, 1, Mods)}), - {reply, ok, State} - end; - -handle_call(stop, _From, State) -> - {stop, normal, ok, State}; - -handle_call(Req, _From, State) -> - lager:error("Bad Request: ~p", [Req]), - {reply, {error, badreq}, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -if_existed(false, Fun) -> Fun(); - -if_existed(_Mod, _Fun) -> {error, already_existed}. - diff --git a/src/emqttd_access_rule.erl b/src/emqttd_access_rule.erl deleted file mode 100644 index f0bad6816..000000000 --- a/src/emqttd_access_rule.erl +++ /dev/null @@ -1,158 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_access_rule). - --author("Feng Lee "). - --include("emqttd.hrl"). - - --type(who() :: all | binary() | - {ipaddr, esockd_cidr:cidr_string()} | - {client, binary()} | - {user, binary()}). - --type(access() :: subscribe | publish | pubsub). - --type(topic() :: binary()). - --type(rule() :: {allow, all} | - {allow, who(), access(), list(topic())} | - {deny, all} | - {deny, who(), access(), list(topic())}). - --export_type([rule/0]). - --export([compile/1, match/3]). - --define(ALLOW_DENY(A), ((A =:= allow) orelse (A =:= deny))). - -%% @doc Compile Access Rule. -compile({A, all}) when ?ALLOW_DENY(A) -> - {A, all}; - -compile({A, Who, Access, Topic}) when ?ALLOW_DENY(A) andalso is_binary(Topic) -> - {A, compile(who, Who), Access, [compile(topic, Topic)]}; - -compile({A, Who, Access, TopicFilters}) when ?ALLOW_DENY(A) -> - {A, compile(who, Who), Access, [compile(topic, Topic) || Topic <- TopicFilters]}. - -compile(who, all) -> - all; -compile(who, {ipaddr, CIDR}) -> - {ipaddr, esockd_cidr:parse(CIDR, true)}; -compile(who, {client, all}) -> - {client, all}; -compile(who, {client, ClientId}) -> - {client, bin(ClientId)}; -compile(who, {user, all}) -> - {user, all}; -compile(who, {user, Username}) -> - {user, bin(Username)}; -compile(who, {'and', Conds}) when is_list(Conds) -> - {'and', [compile(who, Cond) || Cond <- Conds]}; -compile(who, {'or', Conds}) when is_list(Conds) -> - {'or', [compile(who, Cond) || Cond <- Conds]}; - -compile(topic, {eq, Topic}) -> - {eq, emqttd_topic:words(bin(Topic))}; -compile(topic, Topic) -> - Words = emqttd_topic:words(bin(Topic)), - case 'pattern?'(Words) of - true -> {pattern, Words}; - false -> Words - end. - -'pattern?'(Words) -> - lists:member(<<"%u">>, Words) - orelse lists:member(<<"%c">>, Words). - -bin(L) when is_list(L) -> - list_to_binary(L); -bin(B) when is_binary(B) -> - B. - -%% @doc Match Access Rule --spec(match(mqtt_client(), topic(), rule()) -> {matched, allow} | {matched, deny} | nomatch). -match(_Client, _Topic, {AllowDeny, all}) when (AllowDeny =:= allow) orelse (AllowDeny =:= deny) -> - {matched, AllowDeny}; -match(Client, Topic, {AllowDeny, Who, _PubSub, TopicFilters}) - when (AllowDeny =:= allow) orelse (AllowDeny =:= deny) -> - case match_who(Client, Who) andalso match_topics(Client, Topic, TopicFilters) of - true -> {matched, AllowDeny}; - false -> nomatch - end. - -match_who(_Client, all) -> - true; -match_who(_Client, {user, all}) -> - true; -match_who(_Client, {client, all}) -> - true; -match_who(#mqtt_client{client_id = ClientId}, {client, ClientId}) -> - true; -match_who(#mqtt_client{username = Username}, {user, Username}) -> - true; -match_who(#mqtt_client{peername = undefined}, {ipaddr, _Tup}) -> - false; -match_who(#mqtt_client{peername = {IP, _}}, {ipaddr, CIDR}) -> - esockd_cidr:match(IP, CIDR); -match_who(Client, {'and', Conds}) when is_list(Conds) -> - lists:foldl(fun(Who, Allow) -> - match_who(Client, Who) andalso Allow - end, true, Conds); -match_who(Client, {'or', Conds}) when is_list(Conds) -> - lists:foldl(fun(Who, Allow) -> - match_who(Client, Who) orelse Allow - end, false, Conds); -match_who(_Client, _Who) -> - false. - -match_topics(_Client, _Topic, []) -> - false; -match_topics(Client, Topic, [{pattern, PatternFilter}|Filters]) -> - TopicFilter = feed_var(Client, PatternFilter), - case match_topic(emqttd_topic:words(Topic), TopicFilter) of - true -> true; - false -> match_topics(Client, Topic, Filters) - end; -match_topics(Client, Topic, [TopicFilter|Filters]) -> - case match_topic(emqttd_topic:words(Topic), TopicFilter) of - true -> true; - false -> match_topics(Client, Topic, Filters) - end. - -match_topic(Topic, {eq, TopicFilter}) -> - Topic =:= TopicFilter; -match_topic(Topic, TopicFilter) -> - emqttd_topic:match(Topic, TopicFilter). - -feed_var(Client, Pattern) -> - feed_var(Client, Pattern, []). -feed_var(_Client, [], Acc) -> - lists:reverse(Acc); -feed_var(Client = #mqtt_client{client_id = undefined}, [<<"%c">>|Words], Acc) -> - feed_var(Client, Words, [<<"%c">>|Acc]); -feed_var(Client = #mqtt_client{client_id = ClientId}, [<<"%c">>|Words], Acc) -> - feed_var(Client, Words, [ClientId |Acc]); -feed_var(Client = #mqtt_client{username = undefined}, [<<"%u">>|Words], Acc) -> - feed_var(Client, Words, [<<"%u">>|Acc]); -feed_var(Client = #mqtt_client{username = Username}, [<<"%u">>|Words], Acc) -> - feed_var(Client, Words, [Username|Acc]); -feed_var(Client, [W|Words], Acc) -> - feed_var(Client, Words, [W|Acc]). - diff --git a/src/emqttd_acl_internal.erl b/src/emqttd_acl_internal.erl deleted file mode 100644 index 9304b9208..000000000 --- a/src/emqttd_acl_internal.erl +++ /dev/null @@ -1,125 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_acl_internal). - --behaviour(emqttd_acl_mod). - --author("Feng Lee "). - --include("emqttd.hrl"). --include("emqttd_cli.hrl"). - --export([all_rules/0]). - -%% ACL callbacks --export([init/1, check_acl/2, reload_acl/1, description/0]). - --define(ACL_RULE_TAB, mqtt_acl_rule). - --record(state, {config}). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -%% @doc Read all rules --spec(all_rules() -> list(emqttd_access_rule:rule())). -all_rules() -> - case ets:lookup(?ACL_RULE_TAB, all_rules) of - [] -> []; - [{_, Rules}] -> Rules - end. - -%%-------------------------------------------------------------------- -%% ACL callbacks -%%-------------------------------------------------------------------- - -%% @doc Init internal ACL --spec(init([File :: string()]) -> {ok, State :: any()}). -init([File]) -> - ets:new(?ACL_RULE_TAB, [set, public, named_table, {read_concurrency, true}]), - State = #state{config = File}, - true = load_rules_from_file(State), - {ok, State}. - -load_rules_from_file(#state{config = AclFile}) -> - {ok, Terms} = file:consult(AclFile), - Rules = [emqttd_access_rule:compile(Term) || Term <- Terms], - lists:foreach(fun(PubSub) -> - ets:insert(?ACL_RULE_TAB, {PubSub, - lists:filter(fun(Rule) -> filter(PubSub, Rule) end, Rules)}) - end, [publish, subscribe]), - ets:insert(?ACL_RULE_TAB, {all_rules, Terms}). - -filter(_PubSub, {allow, all}) -> - true; -filter(_PubSub, {deny, all}) -> - true; -filter(publish, {_AllowDeny, _Who, publish, _Topics}) -> - true; -filter(_PubSub, {_AllowDeny, _Who, pubsub, _Topics}) -> - true; -filter(subscribe, {_AllowDeny, _Who, subscribe, _Topics}) -> - true; -filter(_PubSub, {_AllowDeny, _Who, _, _Topics}) -> - false. - -%% @doc Check ACL --spec(check_acl({Client, PubSub, Topic}, State) -> allow | deny | ignore when - Client :: mqtt_client(), - PubSub :: pubsub(), - Topic :: binary(), - State :: #state{}). -check_acl(_Who, #state{config = undefined}) -> - allow; -check_acl({Client, PubSub, Topic}, #state{}) -> - case match(Client, Topic, lookup(PubSub)) of - {matched, allow} -> allow; - {matched, deny} -> deny; - nomatch -> ignore - end. - -lookup(PubSub) -> - case ets:lookup(?ACL_RULE_TAB, PubSub) of - [] -> []; - [{PubSub, Rules}] -> Rules - end. - -match(_Client, _Topic, []) -> - nomatch; - -match(Client, Topic, [Rule|Rules]) -> - case emqttd_access_rule:match(Client, Topic, Rule) of - nomatch -> match(Client, Topic, Rules); - {matched, AllowDeny} -> {matched, AllowDeny} - end. - -%% @doc Reload ACL --spec(reload_acl(State :: #state{}) -> ok | {error, Reason :: any()}). -reload_acl(#state{config = undefined}) -> - ok; -reload_acl(State) -> - case catch load_rules_from_file(State) of - {'EXIT', Error} -> {error, Error}; - true -> ?PRINT("~s~n", ["reload acl_internal successfully"]), ok - end. - -%% @doc ACL Module Description --spec(description() -> string()). -description() -> - "Internal ACL with etc/acl.conf". - diff --git a/src/emqttd_alarm.erl b/src/emqttd_alarm.erl deleted file mode 100644 index d271cb425..000000000 --- a/src/emqttd_alarm.erl +++ /dev/null @@ -1,140 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_alarm). - --author("Feng Lee "). - --behaviour(gen_event). - --include("emqttd.hrl"). - --define(ALARM_MGR, ?MODULE). - -%% API Function Exports --export([start_link/0, alarm_fun/0, get_alarms/0, - set_alarm/1, clear_alarm/1, - add_alarm_handler/1, add_alarm_handler/2, - delete_alarm_handler/1]). - -%% gen_event callbacks --export([init/1, handle_event/2, handle_call/2, handle_info/2, - terminate/2, code_change/3]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -start_link() -> - start_with(fun(Pid) -> gen_event:add_handler(Pid, ?MODULE, []) end). - -start_with(Fun) -> - case gen_event:start_link({local, ?ALARM_MGR}) of - {ok, Pid} -> Fun(Pid), {ok, Pid}; - Error -> Error - end. - -alarm_fun() -> alarm_fun(false). - -alarm_fun(Bool) -> - fun(alert, _Alarm) when Bool =:= true -> alarm_fun(true); - (alert, Alarm) when Bool =:= false -> set_alarm(Alarm), alarm_fun(true); - (clear, AlarmId) when Bool =:= true -> clear_alarm(AlarmId), alarm_fun(false); - (clear, _AlarmId) when Bool =:= false -> alarm_fun(false) - end. - --spec(set_alarm(mqtt_alarm()) -> ok). -set_alarm(Alarm) when is_record(Alarm, mqtt_alarm) -> - gen_event:notify(?ALARM_MGR, {set_alarm, Alarm}). - --spec(clear_alarm(any()) -> ok). -clear_alarm(AlarmId) when is_binary(AlarmId) -> - gen_event:notify(?ALARM_MGR, {clear_alarm, AlarmId}). - --spec(get_alarms() -> list(mqtt_alarm())). -get_alarms() -> - gen_event:call(?ALARM_MGR, ?MODULE, get_alarms). - -add_alarm_handler(Module) when is_atom(Module) -> - gen_event:add_handler(?ALARM_MGR, Module, []). - -add_alarm_handler(Module, Args) when is_atom(Module) -> - gen_event:add_handler(?ALARM_MGR, Module, Args). - -delete_alarm_handler(Module) when is_atom(Module) -> - gen_event:delete_handler(?ALARM_MGR, Module, []). - -%%-------------------------------------------------------------------- -%% Default Alarm handler -%%-------------------------------------------------------------------- - -init(_) -> {ok, []}. - -handle_event({set_alarm, Alarm = #mqtt_alarm{id = AlarmId, - severity = Severity, - title = Title, - summary = Summary}}, Alarms)-> - TS = os:timestamp(), - Json = mochijson2:encode([{id, AlarmId}, - {severity, Severity}, - {title, iolist_to_binary(Title)}, - {summary, iolist_to_binary(Summary)}, - {ts, emqttd_time:now_secs(TS)}]), - emqttd:publish(alarm_msg(alert, AlarmId, Json)), - {ok, [Alarm#mqtt_alarm{timestamp = TS} | Alarms]}; - -handle_event({clear_alarm, AlarmId}, Alarms) -> - Json = mochijson2:encode([{id, AlarmId}, {ts, emqttd_time:now_secs()}]), - emqttd:publish(alarm_msg(clear, AlarmId, Json)), - {ok, lists:keydelete(AlarmId, 2, Alarms), hibernate}; - -handle_event(_, Alarms)-> - {ok, Alarms}. - -handle_info(_, Alarms) -> - {ok, Alarms}. - -handle_call(get_alarms, Alarms) -> - {ok, Alarms, Alarms}; - -handle_call(_Query, Alarms) -> - {ok, {error, bad_query}, Alarms}. - -terminate(swap, Alarms) -> - {?MODULE, Alarms}; - -terminate(_, _) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -alarm_msg(Type, AlarmId, Json) -> - Msg = emqttd_message:make(alarm, - topic(Type, AlarmId), - iolist_to_binary(Json)), - emqttd_message:set_flag(sys, Msg). - -topic(alert, AlarmId) -> - emqttd_topic:systop(<<"alarms/", AlarmId/binary, "/alert">>); - -topic(clear, AlarmId) -> - emqttd_topic:systop(<<"alarms/", AlarmId/binary, "/clear">>). - diff --git a/src/emqttd_app.erl b/src/emqttd_app.erl deleted file mode 100644 index 9853c488e..000000000 --- a/src/emqttd_app.erl +++ /dev/null @@ -1,242 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_app). - --behaviour(application). - --author("Feng Lee "). - --include("emqttd_cli.hrl"). - --include("emqttd_protocol.hrl"). - -%% Application callbacks --export([start/2, stop/1]). - --export([start_listener/1, stop_listener/1, restart_listener/1]). - --type(listener() :: {atom(), esockd:listen_on(), [esockd:option()]}). - --define(APP, emqttd). - -%%-------------------------------------------------------------------- -%% Application Callbacks -%%-------------------------------------------------------------------- - -start(_Type, _Args) -> - print_banner(), - ekka:start(), - {ok, Sup} = emqttd_sup:start_link(), - start_servers(Sup), - emqttd_cli:load(), - register_acl_mod(), - start_autocluster(), - register(emqttd, self()), - print_vsn(), - {ok, Sup}. - --spec(stop(State :: term()) -> term()). -stop(_State) -> - catch stop_listeners(). - -%%-------------------------------------------------------------------- -%% Print Banner -%%-------------------------------------------------------------------- - -print_banner() -> - ?PRINT("starting ~s on node '~s'~n", [?APP, node()]). - -print_vsn() -> - {ok, Vsn} = application:get_key(vsn), - ?PRINT("~s ~s is running now~n", [?APP, Vsn]). - -%%-------------------------------------------------------------------- -%% Start Servers -%%-------------------------------------------------------------------- - -start_servers(Sup) -> - Servers = [{"emqttd ctl", emqttd_ctl}, - {"emqttd hook", emqttd_hooks}, - {"emqttd router", emqttd_router}, - {"emqttd pubsub", {supervisor, emqttd_pubsub_sup}}, - {"emqttd stats", emqttd_stats}, - {"emqttd metrics", emqttd_metrics}, - {"emqttd pooler", {supervisor, emqttd_pooler}}, - {"emqttd trace", {supervisor, emqttd_trace_sup}}, - {"emqttd client manager", {supervisor, emqttd_cm_sup}}, - {"emqttd session manager", {supervisor, emqttd_sm_sup}}, - {"emqttd session supervisor", {supervisor, emqttd_session_sup}}, - {"emqttd wsclient supervisor", {supervisor, emqttd_ws_client_sup}}, - {"emqttd broker", emqttd_broker}, - {"emqttd alarm", emqttd_alarm}, - {"emqttd mod supervisor", emqttd_mod_sup}, - {"emqttd bridge supervisor", {supervisor, emqttd_bridge_sup_sup}}, - {"emqttd access control", emqttd_access_control}, - {"emqttd system monitor", {supervisor, emqttd_sysmon_sup}}], - [start_server(Sup, Server) || Server <- Servers]. - -start_server(_Sup, {Name, F}) when is_function(F) -> - ?PRINT("~s is starting...", [Name]), - F(), - ?PRINT_MSG("[ok]~n"); - -start_server(Sup, {Name, Server}) -> - ?PRINT("~s is starting...", [Name]), - start_child(Sup, Server), - ?PRINT_MSG("[ok]~n"); - -start_server(Sup, {Name, Server, Opts}) -> - ?PRINT("~s is starting...", [ Name]), - start_child(Sup, Server, Opts), - ?PRINT_MSG("[ok]~n"). - -start_child(Sup, {supervisor, Module}) -> - supervisor:start_child(Sup, supervisor_spec(Module)); - -start_child(Sup, Module) when is_atom(Module) -> - {ok, _ChiId} = supervisor:start_child(Sup, worker_spec(Module)). - -start_child(Sup, {supervisor, Module}, Opts) -> - supervisor:start_child(Sup, supervisor_spec(Module, Opts)); - -start_child(Sup, Module, Opts) when is_atom(Module) -> - supervisor:start_child(Sup, worker_spec(Module, Opts)). - -supervisor_spec(Module) when is_atom(Module) -> - supervisor_spec(Module, start_link, []). - -supervisor_spec(Module, Opts) -> - supervisor_spec(Module, start_link, [Opts]). - -supervisor_spec(M, F, A) -> - {M, {M, F, A}, permanent, infinity, supervisor, [M]}. - -worker_spec(Module) when is_atom(Module) -> - worker_spec(Module, start_link, []). - -worker_spec(Module, Opts) when is_atom(Module) -> - worker_spec(Module, start_link, [Opts]). - -worker_spec(M, F, A) -> - {M, {M, F, A}, permanent, 10000, worker, [M]}. - -%%-------------------------------------------------------------------- -%% Register default ACL File -%%-------------------------------------------------------------------- - -register_acl_mod() -> - case emqttd:env(acl_file) of - {ok, File} -> emqttd_access_control:register_mod(acl, emqttd_acl_internal, [File]); - undefined -> ok - end. - -%%-------------------------------------------------------------------- -%% Autocluster -%%-------------------------------------------------------------------- - -start_autocluster() -> - ekka:callback(prepare, fun emqttd:shutdown/1), - ekka:callback(reboot, fun emqttd:reboot/0), - ekka:autocluster(?APP, fun after_autocluster/0). - -after_autocluster() -> - emqttd_plugins:init(), - emqttd_plugins:load(), - start_listeners(). - -%%-------------------------------------------------------------------- -%% Start Listeners -%%-------------------------------------------------------------------- - -%% @doc Start Listeners of the broker. --spec(start_listeners() -> any()). -start_listeners() -> lists:foreach(fun start_listener/1, emqttd:env(listeners, [])). - -%% Start mqtt listener --spec(start_listener(listener()) -> any()). -start_listener({tcp, ListenOn, Opts}) -> - start_listener('mqtt:tcp', ListenOn, Opts); - -%% Start mqtt(SSL) listener -start_listener({ssl, ListenOn, Opts}) -> - start_listener('mqtt:ssl', ListenOn, Opts); - -%% Start http listener -start_listener({Proto, ListenOn, Opts}) when Proto == http; Proto == ws -> - mochiweb:start_http('mqtt:ws', ListenOn, Opts, {emqttd_ws, handle_request, []}); - -%% Start https listener -start_listener({Proto, ListenOn, Opts}) when Proto == https; Proto == wss -> - mochiweb:start_http('mqtt:wss', ListenOn, Opts, {emqttd_ws, handle_request, []}); - -start_listener({Proto, ListenOn, Opts}) when Proto == api -> - mochiweb:start_http('mqtt:api', ListenOn, Opts, emqttd_http:http_handler()). - -start_listener(Proto, ListenOn, Opts) -> - Env = lists:append(emqttd:env(client, []), emqttd:env(protocol, [])), - MFArgs = {emqttd_client, start_link, [Env]}, - {ok, _} = esockd:open(Proto, ListenOn, merge_sockopts(Opts), MFArgs). - -merge_sockopts(Options) -> - SockOpts = emqttd_misc:merge_opts( - ?MQTT_SOCKOPTS, proplists:get_value(sockopts, Options, [])), - emqttd_misc:merge_opts(Options, [{sockopts, SockOpts}]). - -%%-------------------------------------------------------------------- -%% Stop Listeners -%%-------------------------------------------------------------------- - -%% @doc Stop Listeners -stop_listeners() -> lists:foreach(fun stop_listener/1, emqttd:env(listeners, [])). - - -%% @private -stop_listener({tcp, ListenOn, _Opts}) -> - esockd:close('mqtt:tcp', ListenOn); -stop_listener({ssl, ListenOn, _Opts}) -> - esockd:close('mqtt:ssl', ListenOn); -stop_listener({Proto, ListenOn, _Opts}) when Proto == http; Proto == ws -> - mochiweb:stop_http('mqtt:ws', ListenOn); -stop_listener({Proto, ListenOn, _Opts}) when Proto == https; Proto == wss -> - mochiweb:stop_http('mqtt:wss', ListenOn); -stop_listener({Proto, ListenOn, _Opts}) when Proto == api -> - mochiweb:stop_http('mqtt:api', ListenOn); -stop_listener({Proto, ListenOn, _Opts}) -> - esockd:close(Proto, ListenOn). - -%% @doc Restart Listeners -restart_listener({tcp, ListenOn, _Opts}) -> - esockd:reopen('mqtt:tcp', ListenOn); -restart_listener({ssl, ListenOn, _Opts}) -> - esockd:reopen('mqtt:ssl', ListenOn); -restart_listener({Proto, ListenOn, _Opts}) when Proto == http; Proto == ws -> - mochiweb:restart_http('mqtt:ws', ListenOn); -restart_listener({Proto, ListenOn, _Opts}) when Proto == https; Proto == wss -> - mochiweb:restart_http('mqtt:wss', ListenOn); -restart_listener({Proto, ListenOn, _Opts}) when Proto == api -> - mochiweb:restart_http('mqtt:api', ListenOn); -restart_listener({Proto, ListenOn, _Opts}) -> - esockd:reopen(Proto, ListenOn). - - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). -merge_sockopts_test_() -> - Opts = [{acceptors, 16}, {max_clients, 512}], - ?_assert(merge_sockopts(Opts) == [{sockopts, ?MQTT_SOCKOPTS} | Opts]). - --endif. diff --git a/src/emqttd_auth_mod.erl b/src/emqttd_auth_mod.erl deleted file mode 100644 index a33631b65..000000000 --- a/src/emqttd_auth_mod.erl +++ /dev/null @@ -1,80 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_auth_mod). - --author("Feng Lee "). - --include("emqttd.hrl"). - --export([passwd_hash/2]). - --type(hash_type() :: plain | md5 | sha | sha256 | pbkdf2 | bcrypt). - -%%-------------------------------------------------------------------- -%% Authentication behavihour -%%-------------------------------------------------------------------- - --ifdef(use_specs). - --callback(init(AuthOpts :: list()) -> {ok, State :: any()}). - --callback(check(Client :: mqtt_client(), - Password :: binary(), - State :: any()) - -> ok | | {ok, boolean()} | ignore | {error, string()}). - --callback(description() -> string()). - --else. - --export([behaviour_info/1]). - -behaviour_info(callbacks) -> - [{init, 1}, {check, 3}, {description, 0}]; -behaviour_info(_Other) -> - undefined. - --endif. - -%% @doc Password Hash --spec(passwd_hash(hash_type(), binary() | tuple()) -> binary()). -passwd_hash(plain, Password) -> - Password; -passwd_hash(md5, Password) -> - hexstring(crypto:hash(md5, Password)); -passwd_hash(sha, Password) -> - hexstring(crypto:hash(sha, Password)); -passwd_hash(sha256, Password) -> - hexstring(crypto:hash(sha256, Password)); -passwd_hash(pbkdf2, {Salt, Password, Macfun, Iterations, Dklen}) -> - case pbkdf2:pbkdf2(Macfun, Password, Salt, Iterations, Dklen) of - {ok, Hexstring} -> pbkdf2:to_hex(Hexstring); - {error, Error} -> lager:error("PasswdHash with pbkdf2 error:~p", [Error]), <<>> - end; -passwd_hash(bcrypt, {Salt, Password}) -> - case bcrypt:hashpw(Password, Salt) of - {ok, HashPassword} -> list_to_binary(HashPassword); - {error, Error}-> lager:error("PasswdHash with bcrypt error:~p", [Error]), <<>> - end. - -hexstring(<>) -> - iolist_to_binary(io_lib:format("~32.16.0b", [X])); -hexstring(<>) -> - iolist_to_binary(io_lib:format("~40.16.0b", [X])); -hexstring(<>) -> - iolist_to_binary(io_lib:format("~64.16.0b", [X])). - diff --git a/src/emqttd_base62.erl b/src/emqttd_base62.erl deleted file mode 100644 index 707981e32..000000000 --- a/src/emqttd_base62.erl +++ /dev/null @@ -1,60 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_base62). - --author("Feng Lee "). - --export([encode/1, decode/1]). - -%% @doc Encode an integer to base62 string --spec(encode(non_neg_integer()) -> binary()). -encode(I) when is_integer(I) andalso I > 0 -> - list_to_binary(encode(I, [])). - -encode(I, Acc) when I < 62 -> - [char(I) | Acc]; -encode(I, Acc) -> - encode(I div 62, [char(I rem 62) | Acc]). - -char(I) when I < 10 -> - $0 + I; - -char(I) when I < 36 -> - $A + I - 10; - -char(I) when I < 62 -> - $a + I - 36. - -%% @doc Decode base62 string to an integer --spec(decode(string() | binary()) -> integer()). -decode(B) when is_binary(B) -> - decode(binary_to_list(B)); -decode(S) when is_list(S) -> - decode(S, 0). - -decode([], I) -> - I; -decode([C|S], I) -> - decode(S, I * 62 + byte(C)). - -byte(C) when $0 =< C andalso C =< $9 -> - C - $0; -byte(C) when $A =< C andalso C =< $Z -> - C - $A + 10; -byte(C) when $a =< C andalso C =< $z -> - C - $a + 36. - diff --git a/src/emqttd_boot.erl b/src/emqttd_boot.erl deleted file mode 100644 index 694b5248a..000000000 --- a/src/emqttd_boot.erl +++ /dev/null @@ -1,65 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_boot). - --author("Feng Lee "). - --export([apply_module_attributes/1, all_module_attributes/1]). - -%% only {F, Args}... -apply_module_attributes(Name) -> - [{Module, [apply(Module, F, Args) || {F, Args} <- Attrs]} || - {_App, Module, Attrs} <- all_module_attributes(Name)]. - -%% Copy from rabbit_misc.erl -all_module_attributes(Name) -> - Targets = - lists:usort( - lists:append( - [[{App, Module} || Module <- Modules] || - {App, _, _} <- ignore_lib_apps(application:loaded_applications()), - {ok, Modules} <- [application:get_key(App, modules)]])), - lists:foldl( - fun ({App, Module}, Acc) -> - case lists:append([Atts || {N, Atts} <- module_attributes(Module), - N =:= Name]) of - [] -> Acc; - Atts -> [{App, Module, Atts} | Acc] - end - end, [], Targets). - -%% Copy from rabbit_misc.erl -module_attributes(Module) -> - case catch Module:module_info(attributes) of - {'EXIT', {undef, [{Module, module_info, [attributes], []} | _]}} -> - []; - {'EXIT', Reason} -> - exit(Reason); - V -> - V - end. - -ignore_lib_apps(Apps) -> - LibApps = [kernel, stdlib, sasl, appmon, eldap, erts, - syntax_tools, ssl, crypto, mnesia, os_mon, - inets, goldrush, lager, gproc, runtime_tools, - snmp, otp_mibs, public_key, asn1, ssh, hipe, - common_test, observer, webtool, xmerl, tools, - test_server, compiler, debugger, eunit, et, - wx], - [App || App = {Name, _, _} <- Apps, not lists:member(Name, LibApps)]. - diff --git a/src/emqttd_bridge_sup.erl b/src/emqttd_bridge_sup.erl deleted file mode 100644 index 29b68c199..000000000 --- a/src/emqttd_bridge_sup.erl +++ /dev/null @@ -1,30 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_bridge_sup). - --export([start_link/3]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -%% @doc Start bridge pool supervisor --spec(start_link(atom(), binary(), [emqttd_bridge:option()]) -> {ok, pid()} | {error, term()}). -start_link(Node, Topic, Options) -> - MFA = {emqttd_bridge, start_link, [Node, Topic, Options]}, - emqttd_pool_sup:start_link({bridge, Node, Topic}, random, MFA). - diff --git a/src/emqttd_broker.erl b/src/emqttd_broker.erl deleted file mode 100644 index 798c94a6a..000000000 --- a/src/emqttd_broker.erl +++ /dev/null @@ -1,204 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_broker). - --behaviour(gen_server). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_internal.hrl"). - -%% API Function Exports --export([start_link/0]). - -%% Event API --export([subscribe/1, notify/2]). - -%% Broker API --export([version/0, uptime/0, datetime/0, sysdescr/0, info/0]). - -%% Tick API --export([start_tick/1, stop_tick/1]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {started_at, sys_interval, heartbeat, ticker, version, sysdescr}). - --define(APP, emqttd). - --define(SERVER, ?MODULE). - --define(BROKER_TAB, mqtt_broker). - -%% $SYS Topics of Broker --define(SYSTOP_BROKERS, [ - version, % Broker version - uptime, % Broker uptime - datetime, % Broker local datetime - sysdescr % Broker description -]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -%% @doc Start emqttd broker --spec(start_link() -> {ok, pid()} | ignore | {error, term()}). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -%% @doc Subscribe broker event --spec(subscribe(EventType :: any()) -> ok). -subscribe(EventType) -> - gproc:reg({p, l, {broker, EventType}}). - -%% @doc Notify broker event --spec(notify(EventType :: any(), Event :: any()) -> ok). -notify(EventType, Event) -> - gproc:send({p, l, {broker, EventType}}, {notify, EventType, self(), Event}). - -%% @doc Get broker info --spec(info() -> list(tuple())). -info() -> - [{version, version()}, - {sysdescr, sysdescr()}, - {uptime, uptime()}, - {datetime, datetime()}]. - -%% @doc Get broker version --spec(version() -> string()). -version() -> - {ok, Version} = application:get_key(?APP, vsn), Version. - -%% @doc Get broker description --spec(sysdescr() -> string()). -sysdescr() -> - {ok, Descr} = application:get_key(?APP, description), Descr. - -%% @doc Get broker uptime --spec(uptime() -> string()). -uptime() -> gen_server:call(?SERVER, uptime). - -%% @doc Get broker datetime --spec(datetime() -> string()). -datetime() -> - {{Y, M, D}, {H, MM, S}} = calendar:local_time(), - lists:flatten( - io_lib:format( - "~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S])). - -%% @doc Start a tick timer. -start_tick(Msg) -> - start_tick(emqttd:env(broker_sys_interval, 60000), Msg). - -start_tick(0, _Msg) -> - undefined; -start_tick(Interval, Msg) when Interval > 0 -> - {ok, TRef} = timer:send_interval(Interval, Msg), TRef. - -%% @doc Stop tick timer -stop_tick(undefined) -> - ok; -stop_tick(TRef) -> - timer:cancel(TRef). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([]) -> - emqttd_time:seed(), - ets:new(?BROKER_TAB, [set, public, named_table]), - % Tick - {ok, #state{started_at = os:timestamp(), - heartbeat = start_tick(1000, heartbeat), - version = list_to_binary(version()), - sysdescr = list_to_binary(sysdescr()), - ticker = start_tick(tick)}, hibernate}. - -handle_call(uptime, _From, State) -> - {reply, uptime(State), State}; - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info(heartbeat, State) -> - publish(uptime, list_to_binary(uptime(State))), - publish(datetime, list_to_binary(datetime())), - {noreply, State, hibernate}; - -handle_info(tick, State = #state{version = Version, sysdescr = Descr}) -> - retain(brokers), - retain(version, Version), - retain(sysdescr, Descr), - {noreply, State, hibernate}; - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, #state{heartbeat = Hb, ticker = TRef}) -> - stop_tick(Hb), - stop_tick(TRef), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -retain(brokers) -> - Payload = list_to_binary(string:join([atom_to_list(N) || - N <- ekka_mnesia:running_nodes()], ",")), - Msg = emqttd_message:make(broker, <<"$SYS/brokers">>, Payload), - emqttd:publish(emqttd_message:set_flag(sys, emqttd_message:set_flag(retain, Msg))). - -retain(Topic, Payload) when is_binary(Payload) -> - Msg = emqttd_message:make(broker, emqttd_topic:systop(Topic), Payload), - emqttd:publish(emqttd_message:set_flag(sys, emqttd_message:set_flag(retain, Msg))). - -publish(Topic, Payload) when is_binary(Payload) -> - Msg = emqttd_message:make(broker, emqttd_topic:systop(Topic), Payload), - emqttd:publish(emqttd_message:set_flag(sys, Msg)). - -uptime(#state{started_at = Ts}) -> - Secs = timer:now_diff(os:timestamp(), Ts) div 1000000, - lists:flatten(uptime(seconds, Secs)). - -uptime(seconds, Secs) when Secs < 60 -> - [integer_to_list(Secs), " seconds"]; -uptime(seconds, Secs) -> - [uptime(minutes, Secs div 60), integer_to_list(Secs rem 60), " seconds"]; -uptime(minutes, M) when M < 60 -> - [integer_to_list(M), " minutes, "]; -uptime(minutes, M) -> - [uptime(hours, M div 60), integer_to_list(M rem 60), " minutes, "]; -uptime(hours, H) when H < 24 -> - [integer_to_list(H), " hours, "]; -uptime(hours, H) -> - [uptime(days, H div 24), integer_to_list(H rem 24), " hours, "]; -uptime(days, D) -> - [integer_to_list(D), " days,"]. - diff --git a/src/emqttd_cli.erl b/src/emqttd_cli.erl deleted file mode 100644 index 32a0eb5a0..000000000 --- a/src/emqttd_cli.erl +++ /dev/null @@ -1,613 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_cli). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_cli.hrl"). - --include("emqttd_protocol.hrl"). - --import(lists, [foreach/2]). - --import(proplists, [get_value/2]). - --export([load/0]). - --export([status/1, broker/1, cluster/1, clients/1, sessions/1, - routes/1, topics/1, subscriptions/1, plugins/1, bridges/1, - listeners/1, vm/1, mnesia/1, trace/1, acl/1]). - --define(PROC_INFOKEYS, [status, - memory, - message_queue_len, - total_heap_size, - heap_size, - stack_size, - reductions]). - --define(MAX_LIMIT, 10000). - --define(APP, emqttd). - -load() -> - Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], - [emqttd_ctl:register_cmd(Cmd, {?MODULE, Cmd}, []) || Cmd <- Cmds], - emqttd_cli_config:register_config(). - -is_cmd(Fun) -> - not lists:member(Fun, [init, load, module_info]). - -%%-------------------------------------------------------------------- -%% Commands -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% @doc Node status - -status([]) -> - {InternalStatus, _ProvidedStatus} = init:get_status(), - ?PRINT("Node ~p is ~p~n", [node(), InternalStatus]), - case lists:keysearch(?APP, 1, application:which_applications()) of - false -> - ?PRINT_MSG("emqttd is not running~n"); - {value, {?APP, _Desc, Vsn}} -> - ?PRINT("emqttd ~s is running~n", [Vsn]) - end; -status(_) -> - ?PRINT_CMD("status", "Show broker status"). - -%%-------------------------------------------------------------------- -%% @doc Query broker - -broker([]) -> - Funs = [sysdescr, version, uptime, datetime], - foreach(fun(Fun) -> - ?PRINT("~-10s: ~s~n", [Fun, emqttd_broker:Fun()]) - end, Funs); - -broker(["stats"]) -> - foreach(fun({Stat, Val}) -> - ?PRINT("~-20s: ~w~n", [Stat, Val]) - end, emqttd_stats:getstats()); - -broker(["metrics"]) -> - foreach(fun({Metric, Val}) -> - ?PRINT("~-24s: ~w~n", [Metric, Val]) - end, lists:sort(emqttd_metrics:all())); - -broker(["pubsub"]) -> - Pubsubs = supervisor:which_children(emqttd_pubsub_sup:pubsub_pool()), - foreach(fun({{_, Id}, Pid, _, _}) -> - ProcInfo = erlang:process_info(Pid, ?PROC_INFOKEYS), - ?PRINT("pubsub: ~w~n", [Id]), - foreach(fun({Key, Val}) -> - ?PRINT(" ~-18s: ~w~n", [Key, Val]) - end, ProcInfo) - end, lists:reverse(Pubsubs)); - -broker(_) -> - ?USAGE([{"broker", "Show broker version, uptime and description"}, - {"broker pubsub", "Show process_info of pubsub"}, - {"broker stats", "Show broker statistics of clients, topics, subscribers"}, - {"broker metrics", "Show broker metrics"}]). - -%%-------------------------------------------------------------------- -%% @doc Cluster with other nodes - -cluster(["join", SNode]) -> - case ekka:join(ekka_node:parse_name(SNode)) of - ok -> - ?PRINT_MSG("Join the cluster successfully.~n"), - cluster(["status"]); - ignore -> - ?PRINT_MSG("Ignore.~n"); - {error, Error} -> - ?PRINT("Failed to join the cluster: ~p~n", [Error]) - end; - -cluster(["leave"]) -> - case ekka:leave() of - ok -> - ?PRINT_MSG("Leave the cluster successfully.~n"), - cluster(["status"]); - {error, Error} -> - ?PRINT("Failed to leave the cluster: ~p~n", [Error]) - end; - -cluster(["force-leave", SNode]) -> - case ekka:force_leave(ekka_node:parse_name(SNode)) of - ok -> - ?PRINT_MSG("Remove the node from cluster successfully.~n"), - cluster(["status"]); - ignore -> - ?PRINT_MSG("Ignore.~n"); - {error, Error} -> - ?PRINT("Failed to remove the node from cluster: ~p~n", [Error]) - end; - -cluster(["status"]) -> - ?PRINT("Cluster status: ~p~n", [ekka_cluster:status()]); - -cluster(_) -> - ?USAGE([{"cluster join ", "Join the cluster"}, - {"cluster leave", "Leave the cluster"}, - {"cluster force-leave ","Force the node leave from cluster"}, - {"cluster status", "Cluster status"}]). - -%%-------------------------------------------------------------------- -%% @doc ACL reload - -acl(["reload"]) -> emqttd_access_control:reload_acl(); -acl(_) -> ?USAGE([{"acl reload", "reload etc/acl.conf"}]). - -%%-------------------------------------------------------------------- -%% @doc Query clients - -clients(["list"]) -> - dump(mqtt_client); - -clients(["show", ClientId]) -> - if_client(ClientId, fun print/1); - -clients(["kick", ClientId]) -> - if_client(ClientId, fun(#mqtt_client{client_pid = Pid}) -> emqttd_client:kick(Pid) end); - -clients(_) -> - ?USAGE([{"clients list", "List all clients"}, - {"clients show ", "Show a client"}, - {"clients kick ", "Kick out a client"}]). - -if_client(ClientId, Fun) -> - case emqttd_cm:lookup(bin(ClientId)) of - undefined -> ?PRINT_MSG("Not Found.~n"); - Client -> Fun(Client) - end. - -%%-------------------------------------------------------------------- -%% @doc Sessions Command - -sessions(["list"]) -> - dump(mqtt_local_session); - -%% performance issue? - -sessions(["list", "persistent"]) -> - lists:foreach(fun print/1, ets:match_object(mqtt_local_session, {'_', '_', false, '_'})); - -%% performance issue? - -sessions(["list", "transient"]) -> - lists:foreach(fun print/1, ets:match_object(mqtt_local_session, {'_', '_', true, '_'})); - -sessions(["show", ClientId]) -> - case ets:lookup(mqtt_local_session, bin(ClientId)) of - [] -> ?PRINT_MSG("Not Found.~n"); - [SessInfo] -> print(SessInfo) - end; - -sessions(_) -> - ?USAGE([{"sessions list", "List all sessions"}, - {"sessions list persistent", "List all persistent sessions"}, - {"sessions list transient", "List all transient sessions"}, - {"sessions show ", "Show a session"}]). - -%%-------------------------------------------------------------------- -%% @doc Routes Command - -routes(["list"]) -> - Routes = emqttd_router:dump(), - foreach(fun print/1, Routes); - -routes(["show", Topic]) -> - Routes = lists:append(ets:lookup(mqtt_route, bin(Topic)), - ets:lookup(mqtt_local_route, bin(Topic))), - foreach(fun print/1, Routes); - -routes(_) -> - ?USAGE([{"routes list", "List all routes"}, - {"routes show ", "Show a route"}]). - -%%-------------------------------------------------------------------- -%% @doc Topics Command - -topics(["list"]) -> - lists:foreach(fun(Topic) -> ?PRINT("~s~n", [Topic]) end, emqttd:topics()); - -topics(["show", Topic]) -> - print(mnesia:dirty_read(mqtt_route, bin(Topic))); - -topics(_) -> - ?USAGE([{"topics list", "List all topics"}, - {"topics show ", "Show a topic"}]). - -subscriptions(["list"]) -> - lists:foreach(fun(Subscription) -> - print(subscription, Subscription) - end, ets:tab2list(mqtt_subscription)); - -subscriptions(["show", ClientId]) -> - case emqttd:subscriptions(bin(ClientId)) of - [] -> - ?PRINT_MSG("Not Found.~n"); - Subscriptions -> - [print(subscription, Sub) || Sub <- Subscriptions] - end; - -subscriptions(["add", ClientId, Topic, QoS]) -> - if_valid_qos(QoS, fun(IntQos) -> - case emqttd_sm:lookup_session(bin(ClientId)) of - undefined -> - ?PRINT_MSG("Error: Session not found!"); - #mqtt_session{sess_pid = SessPid} -> - {Topic1, Options} = emqttd_topic:parse(bin(Topic)), - emqttd_session:subscribe(SessPid, [{Topic1, [{qos, IntQos}|Options]}]), - ?PRINT_MSG("ok~n") - end - end); - -subscriptions(["del", ClientId, Topic]) -> - case emqttd_sm:lookup_session(bin(ClientId)) of - undefined -> - ?PRINT_MSG("Error: Session not found!"); - #mqtt_session{sess_pid = SessPid} -> - emqttd_session:unsubscribe(SessPid, [emqttd_topic:parse(bin(Topic))]), - ?PRINT_MSG("ok~n") - end; - -subscriptions(_) -> - ?USAGE([{"subscriptions list", "List all subscriptions"}, - {"subscriptions show ", "Show subscriptions of a client"}, - {"subscriptions add ", "Add a static subscription manually"}, - {"subscriptions del ", "Delete a static subscription manually"}]). - -% if_could_print(Tab, Fun) -> -% case mnesia:table_info(Tab, size) of -% Size when Size >= ?MAX_LIMIT -> -% ?PRINT("Could not list, too many ~ss: ~p~n", [Tab, Size]); -% _Size -> -% Keys = mnesia:dirty_all_keys(Tab), -% foreach(fun(Key) -> Fun(ets:lookup(Tab, Key)) end, Keys) -% end. - -if_valid_qos(QoS, Fun) -> - try list_to_integer(QoS) of - Int when ?IS_QOS(Int) -> Fun(Int); - _ -> ?PRINT_MSG("QoS should be 0, 1, 2~n") - catch _:_ -> - ?PRINT_MSG("QoS should be 0, 1, 2~n") - end. - -plugins(["list"]) -> - foreach(fun print/1, emqttd_plugins:list()); - -plugins(["load", Name]) -> - case emqttd_plugins:load(list_to_atom(Name)) of - {ok, StartedApps} -> - ?PRINT("Start apps: ~p~nPlugin ~s loaded successfully.~n", [StartedApps, Name]); - {error, Reason} -> - ?PRINT("load plugin error: ~p~n", [Reason]) - end; - -plugins(["unload", Name]) -> - case emqttd_plugins:unload(list_to_atom(Name)) of - ok -> - ?PRINT("Plugin ~s unloaded successfully.~n", [Name]); - {error, Reason} -> - ?PRINT("unload plugin error: ~p~n", [Reason]) - end; - -plugins(_) -> - ?USAGE([{"plugins list", "Show loaded plugins"}, - {"plugins load ", "Load plugin"}, - {"plugins unload ", "Unload plugin"}]). - -%%-------------------------------------------------------------------- -%% @doc Bridges command - -bridges(["list"]) -> - foreach(fun({Node, Topic, _Pid}) -> - ?PRINT("bridge: ~s--~s-->~s~n", [node(), Topic, Node]) - end, emqttd_bridge_sup_sup:bridges()); - -bridges(["options"]) -> - ?PRINT_MSG("Options:~n"), - ?PRINT_MSG(" prefix = string~n"), - ?PRINT_MSG(" suffix = string~n"), - ?PRINT_MSG(" queue = integer~n"), - ?PRINT_MSG("Example:~n"), - ?PRINT_MSG(" prefix=abc/,suffix=/yxz,queue=1000~n"); - -bridges(["start", SNode, Topic]) -> - case emqttd_bridge_sup_sup:start_bridge(list_to_atom(SNode), list_to_binary(Topic)) of - {ok, _} -> ?PRINT_MSG("bridge is started.~n"); - {error, Error} -> ?PRINT("error: ~p~n", [Error]) - end; - -bridges(["start", SNode, Topic, OptStr]) -> - Opts = parse_opts(bridge, OptStr), - case emqttd_bridge_sup_sup:start_bridge(list_to_atom(SNode), list_to_binary(Topic), Opts) of - {ok, _} -> ?PRINT_MSG("bridge is started.~n"); - {error, Error} -> ?PRINT("error: ~p~n", [Error]) - end; - -bridges(["stop", SNode, Topic]) -> - case emqttd_bridge_sup_sup:stop_bridge(list_to_atom(SNode), list_to_binary(Topic)) of - ok -> ?PRINT_MSG("bridge is stopped.~n"); - {error, Error} -> ?PRINT("error: ~p~n", [Error]) - end; - -bridges(_) -> - ?USAGE([{"bridges list", "List bridges"}, - {"bridges options", "Bridge options"}, - {"bridges start ", "Start a bridge"}, - {"bridges start ", "Start a bridge with options"}, - {"bridges stop ", "Stop a bridge"}]). - -parse_opts(Cmd, OptStr) -> - Tokens = string:tokens(OptStr, ","), - [parse_opt(Cmd, list_to_atom(Opt), Val) - || [Opt, Val] <- [string:tokens(S, "=") || S <- Tokens]]. -parse_opt(bridge, suffix, Suffix) -> - {topic_suffix, bin(Suffix)}; -parse_opt(bridge, prefix, Prefix) -> - {topic_prefix, bin(Prefix)}; -parse_opt(bridge, queue, Len) -> - {max_queue_len, list_to_integer(Len)}; -parse_opt(_Cmd, Opt, _Val) -> - ?PRINT("Bad Option: ~s~n", [Opt]). - -%%-------------------------------------------------------------------- -%% @doc vm command - -vm([]) -> - vm(["all"]); - -vm(["all"]) -> - [vm([Name]) || Name <- ["load", "memory", "process", "io", "ports"]]; - -vm(["load"]) -> - [?PRINT("cpu/~-20s: ~s~n", [L, V]) || {L, V} <- emqttd_vm:loads()]; - -vm(["memory"]) -> - [?PRINT("memory/~-17s: ~w~n", [Cat, Val]) || {Cat, Val} <- erlang:memory()]; - -vm(["process"]) -> - foreach(fun({Name, Key}) -> - ?PRINT("process/~-16s: ~w~n", [Name, erlang:system_info(Key)]) - end, [{limit, process_limit}, {count, process_count}]); - -vm(["io"]) -> - IoInfo = erlang:system_info(check_io), - foreach(fun(Key) -> - ?PRINT("io/~-21s: ~w~n", [Key, get_value(Key, IoInfo)]) - end, [max_fds, active_fds]); - -vm(["ports"]) -> - foreach(fun({Name, Key}) -> - ?PRINT("ports/~-16s: ~w~n", [Name, erlang:system_info(Key)]) - end, [{count, port_count}, {limit, port_limit}]); - -vm(_) -> - ?USAGE([{"vm all", "Show info of Erlang VM"}, - {"vm load", "Show load of Erlang VM"}, - {"vm memory", "Show memory of Erlang VM"}, - {"vm process", "Show process of Erlang VM"}, - {"vm io", "Show IO of Erlang VM"}, - {"vm ports", "Show Ports of Erlang VM"}]). - -%%-------------------------------------------------------------------- -%% @doc mnesia Command - -mnesia([]) -> - mnesia:system_info(); - -mnesia(_) -> - ?PRINT_CMD("mnesia", "Mnesia system info"). - -%%-------------------------------------------------------------------- -%% @doc Trace Command - -trace(["list"]) -> - foreach(fun({{Who, Name}, LogFile}) -> - ?PRINT("trace ~s ~s -> ~s~n", [Who, Name, LogFile]) - end, emqttd_trace:all_traces()); - -trace(["client", ClientId, "off"]) -> - trace_off(client, ClientId); - -trace(["client", ClientId, LogFile]) -> - trace_on(client, ClientId, LogFile); - -trace(["topic", Topic, "off"]) -> - trace_off(topic, Topic); - -trace(["topic", Topic, LogFile]) -> - trace_on(topic, Topic, LogFile); - -trace(_) -> - ?USAGE([{"trace list", "List all traces"}, - {"trace client ","Trace a client"}, - {"trace client off", "Stop tracing a client"}, - {"trace topic ", "Trace a topic"}, - {"trace topic off", "Stop tracing a Topic"}]). - -trace_on(Who, Name, LogFile) -> - case emqttd_trace:start_trace({Who, iolist_to_binary(Name)}, LogFile) of - ok -> - ?PRINT("trace ~s ~s successfully.~n", [Who, Name]); - {error, Error} -> - ?PRINT("trace ~s ~s error: ~p~n", [Who, Name, Error]) - end. - -trace_off(Who, Name) -> - case emqttd_trace:stop_trace({Who, iolist_to_binary(Name)}) of - ok -> - ?PRINT("stop tracing ~s ~s successfully.~n", [Who, Name]); - {error, Error} -> - ?PRINT("stop tracing ~s ~s error: ~p.~n", [Who, Name, Error]) - end. - -%%-------------------------------------------------------------------- -%% @doc Listeners Command - -listeners([]) -> - foreach(fun({{Protocol, ListenOn}, Pid}) -> - Info = [{acceptors, esockd:get_acceptors(Pid)}, - {max_clients, esockd:get_max_clients(Pid)}, - {current_clients,esockd:get_current_clients(Pid)}, - {shutdown_count, esockd:get_shutdown_count(Pid)}], - ?PRINT("listener on ~s:~s~n", [Protocol, esockd:to_string(ListenOn)]), - foreach(fun({Key, Val}) -> - ?PRINT(" ~-16s: ~w~n", [Key, Val]) - end, Info) - end, esockd:listeners()); - -listeners(["start", Proto, ListenOn]) -> - case emqttd_app:start_listener({list_to_atom(Proto), parse_listenon(ListenOn), []}) of - {ok, _Pid} -> - io:format("Start ~s listener on ~s successfully.~n", [Proto, ListenOn]); - {error, Error} -> - io:format("Failed to Start ~s listener on ~s, error:~p~n", [Proto, ListenOn, Error]) - end; - -listeners(["restart", Proto, ListenOn]) -> - case emqttd_app:restart_listener({list_to_atom(Proto), parse_listenon(ListenOn), []}) of - {ok, _Pid} -> - io:format("Restart ~s listener on ~s successfully.~n", [Proto, ListenOn]); - {error, Error} -> - io:format("Failed to restart ~s listener on ~s, error:~p~n", [Proto, ListenOn, Error]) - end; - -listeners(["stop", Proto, ListenOn]) -> - case emqttd_app:stop_listener({list_to_atom(Proto), parse_listenon(ListenOn), []}) of - ok -> - io:format("Stop ~s listener on ~s successfully.~n", [Proto, ListenOn]); - {error, Error} -> - io:format("Failed to stop ~s listener on ~s, error:~p~n", [Proto, ListenOn, Error]) - end; - -listeners(_) -> - ?USAGE([{"listeners", "List listeners"}, - {"listeners restart ", "Restart a listener"}, - {"listeners stop ", "Stop a listener"}]). - -%%-------------------------------------------------------------------- -%% Dump ETS -%%-------------------------------------------------------------------- - -dump(Table) -> - dump(Table, ets:first(Table)). - -dump(_Table, '$end_of_table') -> - ok; - -dump(Table, Key) -> - case ets:lookup(Table, Key) of - [Record] -> print(Record); - [] -> ok - end, - dump(Table, ets:next(Table, Key)). - -print([]) -> - ok; - -print(Routes = [#mqtt_route{topic = Topic} | _]) -> - Nodes = [atom_to_list(Node) || #mqtt_route{node = Node} <- Routes], - ?PRINT("~s -> ~s~n", [Topic, string:join(Nodes, ",")]); - -%% print(Subscriptions = [#mqtt_subscription{subid = ClientId} | _]) -> -%% TopicTable = [io_lib:format("~s:~w", [Topic, Qos]) -%% || #mqtt_subscription{topic = Topic, qos = Qos} <- Subscriptions], -%% ?PRINT("~s -> ~s~n", [ClientId, string:join(TopicTable, ",")]); - -%% print(Topics = [#mqtt_topic{}|_]) -> -%% foreach(fun print/1, Topics); - -print(#mqtt_plugin{name = Name, version = Ver, descr = Descr, active = Active}) -> - ?PRINT("Plugin(~s, version=~s, description=~s, active=~s)~n", - [Name, Ver, Descr, Active]); - -print(#mqtt_client{client_id = ClientId, clean_sess = CleanSess, username = Username, - peername = Peername, connected_at = ConnectedAt}) -> - ?PRINT("Client(~s, clean_sess=~s, username=~s, peername=~s, connected_at=~p)~n", - [ClientId, CleanSess, Username, emqttd_net:format(Peername), - emqttd_time:now_secs(ConnectedAt)]); - -%% print(#mqtt_topic{topic = Topic, flags = Flags}) -> -%% ?PRINT("~s: ~s~n", [Topic, string:join([atom_to_list(F) || F <- Flags], ",")]); -print({route, Routes}) -> - foreach(fun print/1, Routes); -print({local_route, Routes}) -> - foreach(fun print/1, Routes); -print(#mqtt_route{topic = Topic, node = Node}) -> - ?PRINT("~s -> ~s~n", [Topic, Node]); -print({Topic, Node}) -> - ?PRINT("~s -> ~s~n", [Topic, Node]); - -print({ClientId, _ClientPid, _Persistent, SessInfo}) -> - Data = lists:append(SessInfo, emqttd_stats:get_session_stats(ClientId)), - InfoKeys = [clean_sess, - subscriptions, - max_inflight, - inflight_len, - mqueue_len, - mqueue_dropped, - awaiting_rel_len, - deliver_msg, - enqueue_msg, - created_at], - ?PRINT("Session(~s, clean_sess=~s, subscriptions=~w, max_inflight=~w, inflight=~w, " - "mqueue_len=~w, mqueue_dropped=~w, awaiting_rel=~w, " - "deliver_msg=~w, enqueue_msg=~w, created_at=~w)~n", - [ClientId | [format(Key, get_value(Key, Data)) || Key <- InfoKeys]]). - -print(subscription, {Sub, {share, _Share, Topic}}) when is_pid(Sub) -> - ?PRINT("~p -> ~s~n", [Sub, Topic]); -print(subscription, {Sub, Topic}) when is_pid(Sub) -> - ?PRINT("~p -> ~s~n", [Sub, Topic]); -print(subscription, {{SubId, SubPid}, {share, _Share, Topic}}) - when is_binary(SubId), is_pid(SubPid) -> - ?PRINT("~s~p -> ~s~n", [SubId, SubPid, Topic]); -print(subscription, {{SubId, SubPid}, Topic}) - when is_binary(SubId), is_pid(SubPid) -> - ?PRINT("~s~p -> ~s~n", [SubId, SubPid, Topic]); -print(subscription, {Sub, Topic, Props}) -> - print(subscription, {Sub, Topic}), - lists:foreach(fun({K, V}) when is_binary(V) -> - ?PRINT(" ~-8s: ~s~n", [K, V]); - ({K, V}) -> - ?PRINT(" ~-8s: ~w~n", [K, V]); - (K) -> - ?PRINT(" ~-8s: true~n", [K]) - end, Props). - -format(created_at, Val) -> - emqttd_time:now_secs(Val); - -format(_, Val) -> - Val. - -bin(S) -> iolist_to_binary(S). - -parse_listenon(ListenOn) -> - case string:tokens(ListenOn, ":") of - [Port] -> list_to_integer(Port); - [IP, Port] -> {IP, list_to_integer(Port)} - end. diff --git a/src/emqttd_cli_config.erl b/src/emqttd_cli_config.erl deleted file mode 100644 index 3c69c8cbc..000000000 --- a/src/emqttd_cli_config.erl +++ /dev/null @@ -1,362 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module (emqttd_cli_config). - --export ([register_config_cli/0, - register_config/0, - run/1, - set_usage/0, - all_cfgs/0, - get_cfg/2, - get_cfg/3, - read_config/1, - write_config/2]). - --define(APP, emqttd). --define(TAB, emqttd_config). - -register_config() -> - application:start(clique), - F = fun() -> ekka_mnesia:running_nodes() end, - clique:register_node_finder(F), - register_config_cli(), - create_config_tab(). - -create_config_tab() -> - case ets:info(?TAB, name) of - undefined -> - ets:new(?TAB, [named_table, public]), - {ok, PluginsEtcDir} = emqttd:env(plugins_etc_dir), - Files = filelib:wildcard("*.conf", PluginsEtcDir), - lists:foreach(fun(File) -> - [FileName | _] = string:tokens(File, "."), - Configs = cuttlefish_conf:file(lists:concat([PluginsEtcDir, File])), - ets:insert(?TAB, {list_to_atom(FileName), Configs}) - end, Files); - _ -> - ok - end. - -read_config(App) -> - case ets:lookup(?TAB, App) of - [] -> []; - [{_, Value}] -> Value - end. - -write_config(App, Terms) -> - ets:insert(?TAB, {App, Terms}). - -run(Cmd) -> - clique:run(Cmd). - -register_config_cli() -> - ok = clique_config:load_schema([code:priv_dir(?APP)], ?APP), - register_protocol_formatter(), - register_client_formatter(), - register_session_formatter(), - register_queue_formatter(), - register_lager_formatter(), - - register_auth_config(), - register_protocol_config(), - register_connection_config(), - register_client_config(), - register_session_config(), - register_queue_config(), - register_broker_config(), - register_lager_config(). - -set_usage() -> - io:format("~-40s# ~-20s# ~-20s ~p~n", ["key", "value", "datatype", "app"]), - io:format("------------------------------------------------------------------------------------------------~n"), - lists:foreach(fun({Key, Val, Datatype, App}) -> - io:format("~-40s# ~-20s# ~-20s ~p~n", [Key, Val, Datatype, App]) - end, all_cfgs()), - io:format("------------------------------------------------------------------------------------------------~n"), - io:format("Usage: set key=value --app=appname~n"). - -all_cfgs() -> - {Mappings, Mappings1} = lists:foldl( - fun({Key, {_, Map, _}}, {Acc, Acc1}) -> - Map1 = lists:map(fun(M) -> {cuttlefish_mapping:variable(M), Key} end, Map), - {Acc ++ Map, Acc1 ++ Map1} - end, {[], []}, ets:tab2list(clique_schema)), - lists:foldl(fun({Key, _}, Acc) -> - case lists:keyfind(cuttlefish_variable:tokenize(Key), 2, Mappings) of - false -> Acc; - Map -> - Datatype = format_datatype(cuttlefish_mapping:datatype(Map)), - App = proplists:get_value(cuttlefish_variable:tokenize(Key), Mappings1), - [{_, [Val0]}] = clique_config:show([Key], [{app, App}]), - Val = any_to_string(proplists:get_value(Key, Val0)), - [{Key, Val, Datatype, App} | Acc] - end - end, [],lists:sort(ets:tab2list(clique_config))). - -get_cfg(App, Key) -> - get_cfg(App, Key, undefined). - -get_cfg(App, Key, Def) -> - [{_, [Val0]}] = clique_config:show([Key], [{app, App}]), - proplists:get_value(Key, Val0, Def). - -format_datatype(Value) -> - format_datatype(Value, ""). - -format_datatype([Head], Acc) when is_tuple(Head) -> - [Head1 | _] = erlang:tuple_to_list(Head), - lists:concat([Acc, Head1]); -format_datatype([Head], Acc) -> - lists:concat([Acc, Head]); -format_datatype([Head | Tail], Acc) when is_tuple(Head)-> - [Head1 | _] = erlang:tuple_to_list(Head), - format_datatype(Tail, Acc ++ lists:concat([Head1, ", "])); -format_datatype([Head | Tail], Acc) -> - format_datatype(Tail, Acc ++ lists:concat([Head, ", "])). - -%%-------------------------------------------------------------------- -%% Auth/Acl -%%-------------------------------------------------------------------- - -register_auth_config() -> - ConfigKeys = ["mqtt.allow_anonymous", - "mqtt.acl_nomatch", - "mqtt.acl_file", - "mqtt.cache_acl"], - [clique:register_config(Key , fun auth_config_callback/2) || Key <- ConfigKeys], - ok = register_config_whitelist(ConfigKeys). - -auth_config_callback([_, KeyStr], Value) -> - application:set_env(?APP, l2a(KeyStr), Value), " successfully\n". - -%%-------------------------------------------------------------------- -%% MQTT Protocol -%%-------------------------------------------------------------------- - -register_protocol_formatter() -> - ConfigKeys = ["max_clientid_len", - "max_packet_size", - "websocket_protocol_header", - "keepalive_backoff"], - [clique:register_formatter(["mqtt", Key], fun protocol_formatter_callback/2) || Key <- ConfigKeys]. - -protocol_formatter_callback([_, "websocket_protocol_header"], Params) -> - Params; -protocol_formatter_callback([_, Key], Params) -> - proplists:get_value(l2a(Key), Params). - -register_protocol_config() -> - ConfigKeys = ["mqtt.max_clientid_len", - "mqtt.max_packet_size", - "mqtt.websocket_protocol_header", - "mqtt.keepalive_backoff"], - [clique:register_config(Key , fun protocol_config_callback/2) || Key <- ConfigKeys], - ok = register_config_whitelist(ConfigKeys). - -protocol_config_callback([_AppStr, KeyStr], Value) -> - protocol_config_callback(protocol, l2a(KeyStr), Value). -protocol_config_callback(_App, websocket_protocol_header, Value) -> - application:set_env(?APP, websocket_protocol_header, Value), - " successfully\n"; -protocol_config_callback(App, Key, Value) -> - {ok, Env} = emqttd:env(App), - application:set_env(?APP, App, lists:keyreplace(Key, 1, Env, {Key, Value})), - " successfully\n". - -%%-------------------------------------------------------------------- -%% MQTT Connection -%%-------------------------------------------------------------------- - -register_connection_config() -> - ConfigKeys = ["mqtt.conn.force_gc_count"], - [clique:register_config(Key , fun connection_config_callback/2) || Key <- ConfigKeys], - ok = register_config_whitelist(ConfigKeys). - -connection_config_callback([_, KeyStr0, KeyStr1], Value) -> - KeyStr = lists:concat([KeyStr0, "_", KeyStr1]), - application:set_env(?APP, l2a(KeyStr), Value), - " successfully\n". - -%%-------------------------------------------------------------------- -%% MQTT Client -%%-------------------------------------------------------------------- - -register_client_formatter() -> - ConfigKeys = ["max_publish_rate", - "idle_timeout", - "enable_stats"], - [clique:register_formatter(["mqtt", "client", Key], fun client_formatter_callback/2) || Key <- ConfigKeys]. - -client_formatter_callback([_, _, Key], Params) -> - proplists:get_value(list_to_atom(Key), Params). - -register_client_config() -> - ConfigKeys = ["mqtt.client.max_publish_rate", - "mqtt.client.idle_timeout", - "mqtt.client.enable_stats"], - [clique:register_config(Key , fun client_config_callback/2) || Key <- ConfigKeys], - ok = register_config_whitelist(ConfigKeys). - -client_config_callback([_, AppStr, KeyStr], Value) -> - client_config_callback(l2a(AppStr), l2a(KeyStr), Value). - -client_config_callback(App, idle_timeout, Value) -> - {ok, Env} = emqttd:env(App), - application:set_env(?APP, App, lists:keyreplace(client_idle_timeout, 1, Env, {client_idle_timeout, Value})), - " successfully\n"; -client_config_callback(App, enable_stats, Value) -> - {ok, Env} = emqttd:env(App), - application:set_env(?APP, App, lists:keyreplace(client_enable_stats, 1, Env, {client_enable_stats, Value})), - " successfully\n"; -client_config_callback(App, Key, Value) -> - {ok, Env} = emqttd:env(App), - application:set_env(?APP, App, lists:keyreplace(Key, 1, Env, {Key, Value})), - " successfully\n". - -%%-------------------------------------------------------------------- -%% session -%%-------------------------------------------------------------------- - -register_session_formatter() -> - ConfigKeys = ["max_subscriptions", - "upgrade_qos", - "max_inflight", - "retry_interval", - "max_awaiting_rel", - "await_rel_timeout", - "enable_stats", - "expiry_interval", - "ignore_loop_deliver"], - [clique:register_formatter(["mqtt", "session", Key], fun session_formatter_callback/2) || Key <- ConfigKeys]. - -session_formatter_callback([_, _, Key], Params) -> - proplists:get_value(list_to_atom(Key), Params). - -register_session_config() -> - ConfigKeys = ["mqtt.session.max_subscriptions", - "mqtt.session.upgrade_qos", - "mqtt.session.max_inflight", - "mqtt.session.retry_interval", - "mqtt.session.max_awaiting_rel", - "mqtt.session.await_rel_timeout", - "mqtt.session.enable_stats", - "mqtt.session.expiry_interval", - "mqtt.session.ignore_loop_deliver"], - [clique:register_config(Key , fun session_config_callback/2) || Key <- ConfigKeys], - ok = register_config_whitelist(ConfigKeys). - -session_config_callback([_, AppStr, KeyStr], Value) -> - session_config_callback(l2a(AppStr), l2a(KeyStr), Value). -session_config_callback(App, Key, Value) -> - {ok, Env} = emqttd:env(App), - application:set_env(?APP, App, lists:keyreplace(Key, 1, Env, {Key, Value})), - " successfully\n". - -l2a(List) -> list_to_atom(List). - -%%-------------------------------------------------------------------- -%% MQTT MQueue -%%-------------------------------------------------------------------- - -register_queue_formatter() -> - ConfigKeys = ["type", - "priority", - "max_length", - "low_watermark", - "high_watermark", - "store_qos0"], - [clique:register_formatter(["mqtt", "mqueue", Key], fun queue_formatter_callback/2) || Key <- ConfigKeys]. - -queue_formatter_callback([_, _, Key], Params) -> - proplists:get_value(list_to_atom(Key), Params). - -register_queue_config() -> - ConfigKeys = ["mqtt.mqueue.type", - "mqtt.mqueue.priority", - "mqtt.mqueue.max_length", - "mqtt.mqueue.low_watermark", - "mqtt.mqueue.high_watermark", - "mqtt.mqueue.store_qos0"], - [clique:register_config(Key , fun queue_config_callback/2) || Key <- ConfigKeys], - ok = register_config_whitelist(ConfigKeys). - -queue_config_callback([_, AppStr, KeyStr], Value) -> - queue_config_callback(l2a(AppStr), l2a(KeyStr), Value). - -queue_config_callback(App, low_watermark, Value) -> - {ok, Env} = emqttd:env(App), - application:set_env(?APP, App, lists:keyreplace(low_watermark, 1, Env, {low_watermark, Value})), - " successfully\n"; -queue_config_callback(App, high_watermark, Value) -> - {ok, Env} = emqttd:env(App), - application:set_env(?APP, App, lists:keyreplace(high_watermark, 1, Env, {high_watermark, Value})), - " successfully\n"; -queue_config_callback(App, Key, Value) -> - {ok, Env} = emqttd:env(App), - application:set_env(?APP, App, lists:keyreplace(Key, 1, Env, {Key, Value})), - " successfully\n". - -%%-------------------------------------------------------------------- -%% MQTT Broker -%%-------------------------------------------------------------------- - -register_broker_config() -> - ConfigKeys = ["mqtt.broker.sys_interval"], - [clique:register_config(Key , fun broker_config_callback/2) || Key <- ConfigKeys], - ok = register_config_whitelist(ConfigKeys). - -broker_config_callback([_, KeyStr0, KeyStr1], Value) -> - KeyStr = lists:concat([KeyStr0, "_", KeyStr1]), - application:set_env(?APP, l2a(KeyStr), Value), - " successfully\n". - -%%-------------------------------------------------------------------- -%% MQTT Lager -%%-------------------------------------------------------------------- - -register_lager_formatter() -> - ConfigKeys = ["level"], - [clique:register_formatter(["log", "console", Key], fun lager_formatter_callback/2) || Key <- ConfigKeys]. - -lager_formatter_callback(_, Params) -> - proplists:get_value(lager_console_backend, Params). - -register_lager_config() -> - ConfigKeys = ["log.console.level"], - [clique:register_config(Key , fun lager_config_callback/2) || Key <- ConfigKeys], - ok = register_config_whitelist(ConfigKeys). - -lager_config_callback(_, Value) -> - lager:set_loglevel(lager_console_backend, Value), - " successfully\n". - -register_config_whitelist(ConfigKeys) -> - clique:register_config_whitelist(ConfigKeys, ?APP). - -%%-------------------------------------------------------------------- -%% Inner Function -%%-------------------------------------------------------------------- -any_to_string(I) when is_integer(I) -> - integer_to_list(I); -any_to_string(F) when is_float(F)-> - float_to_list(F,[{decimals, 4}]); -any_to_string(A) when is_atom(A) -> - atom_to_list(A); -any_to_string(B) when is_binary(B) -> - binary_to_list(B); -any_to_string(L) when is_list(L) -> - L. diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl deleted file mode 100644 index f479d2253..000000000 --- a/src/emqttd_client.erl +++ /dev/null @@ -1,396 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc MQTT/TCP Connection. - --module(emqttd_client). - --behaviour(gen_server2). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --include("emqttd_internal.hrl"). - --import(proplists, [get_value/2, get_value/3]). - -%% API Function Exports --export([start_link/2]). - -%% Management and Monitor API --export([info/1, stats/1, kick/1, clean_acl_cache/2]). - --export([set_rate_limit/2, get_rate_limit/1]). - -%% SUB/UNSUB Asynchronously. Called by plugins. --export([subscribe/2, unsubscribe/2]). - -%% Get the session proc? --export([session/1]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - code_change/3, terminate/2]). - -%% gen_server2 Callbacks --export([prioritise_call/4, prioritise_info/3, handle_pre_hibernate/1]). - -%% Client State -%% Unused fields: connname, peerhost, peerport --record(client_state, {connection, peername, conn_state, await_recv, - rate_limit, packet_size, parser, proto_state, - keepalive, enable_stats, idle_timeout, force_gc_count}). - --define(INFO_KEYS, [peername, conn_state, await_recv]). - --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). - --define(LOG(Level, Format, Args, State), - lager:Level("Client(~s): " ++ Format, - [esockd_net:format(State#client_state.peername) | Args])). - -start_link(Conn, Env) -> - {ok, proc_lib:spawn_link(?MODULE, init, [[Conn, Env]])}. - -info(CPid) -> - gen_server2:call(CPid, info). - -stats(CPid) -> - gen_server2:call(CPid, stats). - -kick(CPid) -> - gen_server2:call(CPid, kick). - -set_rate_limit(Cpid, Rl) -> - gen_server2:call(Cpid, {set_rate_limit, Rl}). - -get_rate_limit(Cpid) -> - gen_server2:call(Cpid, get_rate_limit). - -subscribe(CPid, TopicTable) -> - CPid ! {subscribe, TopicTable}. - -unsubscribe(CPid, Topics) -> - CPid ! {unsubscribe, Topics}. - -session(CPid) -> - gen_server2:call(CPid, session, infinity). - -clean_acl_cache(CPid, Topic) -> - gen_server2:call(CPid, {clean_acl_cache, Topic}). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([Conn0, Env]) -> - {ok, Conn} = Conn0:wait(), - case Conn:peername() of - {ok, Peername} -> do_init(Conn, Env, Peername); - {error, enotconn} -> Conn:fast_close(), - exit(normal); - {error, Reason} -> Conn:fast_close(), - exit({shutdown, Reason}) - end. - -do_init(Conn, Env, Peername) -> - %% Send Fun - SendFun = send_fun(Conn, Peername), - RateLimit = get_value(rate_limit, Conn:opts()), - PacketSize = get_value(max_packet_size, Env, ?MAX_PACKET_SIZE), - Parser = emqttd_parser:initial_state(PacketSize), - ProtoState = emqttd_protocol:init(Conn, Peername, SendFun, Env), - EnableStats = get_value(client_enable_stats, Env, false), - IdleTimout = get_value(client_idle_timeout, Env, 30000), - ForceGcCount = emqttd_gc:conn_max_gc_count(), - State = run_socket(#client_state{connection = Conn, - peername = Peername, - await_recv = false, - conn_state = running, - rate_limit = RateLimit, - packet_size = PacketSize, - parser = Parser, - proto_state = ProtoState, - enable_stats = EnableStats, - idle_timeout = IdleTimout, - force_gc_count = ForceGcCount}), - gen_server2:enter_loop(?MODULE, [], State, self(), IdleTimout, - {backoff, 2000, 2000, 20000}). - -send_fun(Conn, Peername) -> - Self = self(), - fun(Packet) -> - Data = emqttd_serializer:serialize(Packet), - ?LOG(debug, "SEND ~p", [Data], #client_state{peername = Peername}), - emqttd_metrics:inc('bytes/sent', iolist_size(Data)), - try Conn:async_send(Data) of - ok -> ok; - true -> ok; %% Compatible with esockd 4.x - {error, Reason} -> Self ! {shutdown, Reason} - catch - error:Error -> Self ! {shutdown, Error} - end - end. - -prioritise_call(Msg, _From, _Len, _State) -> - case Msg of info -> 10; stats -> 10; state -> 10; _ -> 5 end. - -prioritise_info(Msg, _Len, _State) -> - case Msg of {redeliver, _} -> 5; _ -> 0 end. - -handle_pre_hibernate(State) -> - {hibernate, emqttd_gc:reset_conn_gc_count(#client_state.force_gc_count, emit_stats(State))}. - -handle_call(info, From, State = #client_state{proto_state = ProtoState}) -> - ProtoInfo = emqttd_protocol:info(ProtoState), - ClientInfo = ?record_to_proplist(client_state, State, ?INFO_KEYS), - {reply, Stats, _, _} = handle_call(stats, From, State), - reply(lists:append([ClientInfo, ProtoInfo, Stats]), State); - -handle_call(stats, _From, State = #client_state{proto_state = ProtoState}) -> - reply(lists:append([emqttd_misc:proc_stats(), - emqttd_protocol:stats(ProtoState), - sock_stats(State)]), State); - -handle_call(kick, _From, State) -> - {stop, {shutdown, kick}, ok, State}; - -handle_call({set_rate_limit, Rl}, _From, State) -> - reply(ok, State#client_state{rate_limit = Rl}); - -handle_call(get_rate_limit, _From, State = #client_state{rate_limit = Rl}) -> - reply(Rl, State); - -handle_call(session, _From, State = #client_state{proto_state = ProtoState}) -> - reply(emqttd_protocol:session(ProtoState), State); - -handle_call({clean_acl_cache, Topic}, _From, State) -> - erase({acl, publish, Topic}), - reply(ok, State); - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info({subscribe, TopicTable}, State) -> - with_proto( - fun(ProtoState) -> - emqttd_protocol:subscribe(TopicTable, ProtoState) - end, State); - -handle_info({unsubscribe, Topics}, State) -> - with_proto( - fun(ProtoState) -> - emqttd_protocol:unsubscribe(Topics, ProtoState) - end, State); - -%% Asynchronous SUBACK -handle_info({suback, PacketId, GrantedQos}, State) -> - with_proto( - fun(ProtoState) -> - Packet = ?SUBACK_PACKET(PacketId, GrantedQos), - emqttd_protocol:send(Packet, ProtoState) - end, State); - -handle_info({deliver, Message}, State) -> - with_proto( - fun(ProtoState) -> - emqttd_protocol:send(Message, ProtoState) - end, State); - -handle_info({redeliver, {?PUBREL, PacketId}}, State) -> - with_proto( - fun(ProtoState) -> - emqttd_protocol:pubrel(PacketId, ProtoState) - end, State); - -handle_info(emit_stats, State) -> - {noreply, emit_stats(State), hibernate}; - -handle_info(timeout, State) -> - shutdown(idle_timeout, State); - -%% Fix issue #535 -handle_info({shutdown, Error}, State) -> - shutdown(Error, State); - -handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> - ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid], State), - shutdown(conflict, State); - -handle_info(activate_sock, State) -> - {noreply, run_socket(State#client_state{conn_state = running}), hibernate}; - -handle_info({inet_async, _Sock, _Ref, {ok, Data}}, State) -> - Size = iolist_size(Data), - ?LOG(debug, "RECV ~p", [Data], State), - emqttd_metrics:inc('bytes/received', Size), - received(Data, rate_limit(Size, State#client_state{await_recv = false})); - -handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> - shutdown(Reason, State); - -handle_info({inet_reply, _Sock, ok}, State) -> - {noreply, gc(State), hibernate}; %% Tune GC - -handle_info({inet_reply, _Sock, {error, Reason}}, State) -> - shutdown(Reason, State); - -handle_info({keepalive, start, Interval}, State = #client_state{connection = Conn}) -> - ?LOG(debug, "Keepalive at the interval of ~p", [Interval], State), - StatFun = fun() -> - case Conn:getstat([recv_oct]) of - {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; - {error, Error} -> {error, Error} - end - end, - case emqttd_keepalive:start(StatFun, Interval, {keepalive, check}) of - {ok, KeepAlive} -> - {noreply, State#client_state{keepalive = KeepAlive}, hibernate}; - {error, Error} -> - ?LOG(warning, "Keepalive error - ~p", [Error], State), - shutdown(Error, State) - end; - -handle_info({keepalive, check}, State = #client_state{keepalive = KeepAlive}) -> - case emqttd_keepalive:check(KeepAlive) of - {ok, KeepAlive1} -> - {noreply, State#client_state{keepalive = KeepAlive1}, hibernate}; - {error, timeout} -> - ?LOG(debug, "Keepalive timeout", [], State), - shutdown(keepalive_timeout, State); - {error, Error} -> - ?LOG(warning, "Keepalive error - ~p", [Error], State), - shutdown(Error, State) - end; - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(Reason, State = #client_state{connection = Conn, - keepalive = KeepAlive, - proto_state = ProtoState}) -> - - ?LOG(debug, "Terminated for ~p", [Reason], State), - Conn:fast_close(), - emqttd_keepalive:cancel(KeepAlive), - case {ProtoState, Reason} of - {undefined, _} -> - ok; - {_, {shutdown, Error}} -> - emqttd_protocol:shutdown(Error, ProtoState); - {_, Reason} -> - emqttd_protocol:shutdown(Reason, ProtoState) - end. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -%% Receive and Parse TCP Data -received(<<>>, State) -> - {noreply, gc(State), hibernate}; - -received(Bytes, State = #client_state{parser = Parser, - packet_size = PacketSize, - proto_state = ProtoState, - idle_timeout = IdleTimeout}) -> - case catch emqttd_parser:parse(Bytes, Parser) of - {more, NewParser} -> - {noreply, run_socket(State#client_state{parser = NewParser}), IdleTimeout}; - {ok, Packet, Rest} -> - emqttd_metrics:received(Packet), - case emqttd_protocol:received(Packet, ProtoState) of - {ok, ProtoState1} -> - received(Rest, State#client_state{parser = emqttd_parser:initial_state(PacketSize), - proto_state = ProtoState1}); - {error, Error} -> - ?LOG(error, "Protocol error - ~p", [Error], State), - shutdown(Error, State); - {error, Error, ProtoState1} -> - shutdown(Error, State#client_state{proto_state = ProtoState1}); - {stop, Reason, ProtoState1} -> - stop(Reason, State#client_state{proto_state = ProtoState1}) - end; - {error, Error} -> - ?LOG(error, "Framing error - ~p", [Error], State), - shutdown(Error, State); - {'EXIT', Reason} -> - ?LOG(error, "Parser failed for ~p", [Reason], State), - ?LOG(error, "Error data: ~p", [Bytes], State), - shutdown(parser_error, State) - end. - -rate_limit(_Size, State = #client_state{rate_limit = undefined}) -> - run_socket(State); -rate_limit(Size, State = #client_state{rate_limit = Rl}) -> - case Rl:check(Size) of - {0, Rl1} -> - run_socket(State#client_state{conn_state = running, rate_limit = Rl1}); - {Pause, Rl1} -> - ?LOG(warning, "Rate limiter pause for ~p", [Pause], State), - erlang:send_after(Pause, self(), activate_sock), - State#client_state{conn_state = blocked, rate_limit = Rl1} - end. - -run_socket(State = #client_state{conn_state = blocked}) -> - State; -run_socket(State = #client_state{await_recv = true}) -> - State; -run_socket(State = #client_state{connection = Conn}) -> - Conn:async_recv(0, infinity), - State#client_state{await_recv = true}. - -with_proto(Fun, State = #client_state{proto_state = ProtoState}) -> - {ok, ProtoState1} = Fun(ProtoState), - {noreply, State#client_state{proto_state = ProtoState1}, hibernate}. - -emit_stats(State = #client_state{proto_state = ProtoState}) -> - emit_stats(emqttd_protocol:clientid(ProtoState), State). - -emit_stats(_ClientId, State = #client_state{enable_stats = false}) -> - State; -emit_stats(undefined, State) -> - State; -emit_stats(ClientId, State) -> - {reply, Stats, _, _} = handle_call(stats, undefined, State), - emqttd_stats:set_client_stats(ClientId, Stats), - State. - -sock_stats(#client_state{connection = Conn}) -> - case Conn:getstat(?SOCK_STATS) of {ok, Ss} -> Ss; {error, _} -> [] end. - -reply(Reply, State) -> - {reply, Reply, State, hibernate}. - -shutdown(Reason, State) -> - stop({shutdown, Reason}, State). - -stop(Reason, State) -> - {stop, Reason, State}. - -gc(State = #client_state{connection = Conn}) -> - Cb = fun() -> Conn:gc(), emit_stats(State) end, - emqttd_gc:maybe_force_gc(#client_state.force_gc_count, State, Cb). - diff --git a/src/emqttd_cm.erl b/src/emqttd_cm.erl deleted file mode 100644 index bcaf353ed..000000000 --- a/src/emqttd_cm.erl +++ /dev/null @@ -1,160 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc MQTT Client Manager - --module(emqttd_cm). - --behaviour(gen_server2). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_internal.hrl"). - -%% API Exports --export([start_link/3]). - --export([lookup/1, lookup_proc/1, reg/1, unreg/1]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - -%% gen_server2 priorities --export([prioritise_call/4, prioritise_cast/3, prioritise_info/3]). - --record(state, {pool, id, statsfun, monitors}). - --define(POOL, ?MODULE). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -%% @doc Start Client Manager --spec(start_link(atom(), pos_integer(), fun()) -> {ok, pid()} | ignore | {error, term()}). -start_link(Pool, Id, StatsFun) -> - gen_server2:start_link(?MODULE, [Pool, Id, StatsFun], []). - -%% @doc Lookup Client by ClientId --spec(lookup(binary()) -> mqtt_client() | undefined). -lookup(ClientId) when is_binary(ClientId) -> - case ets:lookup(mqtt_client, ClientId) of [Client] -> Client; [] -> undefined end. - -%% @doc Lookup client pid by clientId --spec(lookup_proc(binary()) -> pid() | undefined). -lookup_proc(ClientId) when is_binary(ClientId) -> - try ets:lookup_element(mqtt_client, ClientId, #mqtt_client.client_pid) - catch - error:badarg -> undefined - end. - -%% @doc Register ClientId with Pid. --spec(reg(mqtt_client()) -> ok). -reg(Client = #mqtt_client{client_id = ClientId}) -> - gen_server2:call(pick(ClientId), {reg, Client}, 120000). - -%% @doc Unregister clientId with pid. --spec(unreg(binary()) -> ok). -unreg(ClientId) when is_binary(ClientId) -> - gen_server2:cast(pick(ClientId), {unreg, ClientId, self()}). - -pick(ClientId) -> gproc_pool:pick_worker(?POOL, ClientId). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id, StatsFun]) -> - ?GPROC_POOL(join, Pool, Id), - {ok, #state{pool = Pool, id = Id, statsfun = StatsFun, monitors = dict:new()}}. - -prioritise_call(Req, _From, _Len, _State) -> - case Req of {reg, _Client} -> 2; _ -> 1 end. - -prioritise_cast(Msg, _Len, _State) -> - case Msg of {unreg, _ClientId, _Pid} -> 9; _ -> 1 end. - -prioritise_info(_Msg, _Len, _State) -> - 3. - -handle_call({reg, Client = #mqtt_client{client_id = ClientId, - client_pid = Pid}}, _From, State) -> - case lookup_proc(ClientId) of - Pid -> - {reply, ok, State}; - _ -> - ets:insert(mqtt_client, Client), - {reply, ok, setstats(monitor_client(ClientId, Pid, State))} - end; - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast({unreg, ClientId, Pid}, State) -> - case lookup_proc(ClientId) of - Pid -> - ets:delete(mqtt_client, ClientId), - {noreply, setstats(State)}; - _ -> - {noreply, State} - end; - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info({'DOWN', MRef, process, DownPid, _Reason}, State) -> - case dict:find(MRef, State#state.monitors) of - {ok, {ClientId, DownPid}} -> - case lookup_proc(ClientId) of - DownPid -> - emqttd_stats:del_client_stats(ClientId), - ets:delete(mqtt_client, ClientId); - _ -> - ignore - end, - {noreply, setstats(erase_monitor(MRef, State))}; - error -> - lager:error("MRef of client ~p not found", [DownPid]), - {noreply, State} - end; - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - ?GPROC_POOL(leave, Pool, Id), ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -monitor_client(ClientId, Pid, State = #state{monitors = Monitors}) -> - MRef = erlang:monitor(process, Pid), - State#state{monitors = dict:store(MRef, {ClientId, Pid}, Monitors)}. - -erase_monitor(MRef, State = #state{monitors = Monitors}) -> - erlang:demonitor(MRef, [flush]), - State#state{monitors = dict:erase(MRef, Monitors)}. - -setstats(State = #state{statsfun = StatsFun}) -> - StatsFun(ets:info(mqtt_client, size)), State. - diff --git a/src/emqttd_cm_sup.erl b/src/emqttd_cm_sup.erl deleted file mode 100644 index ccaea00fb..000000000 --- a/src/emqttd_cm_sup.erl +++ /dev/null @@ -1,58 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Client Manager Supervisor. - --module(emqttd_cm_sup). - --behaviour(supervisor). - --author("Feng Lee "). - --include("emqttd.hrl"). - -%% API --export([start_link/0]). - -%% Supervisor callbacks --export([init/1]). - --define(CM, emqttd_cm). - --define(TAB, mqtt_client). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - %% Create client table - create_client_tab(), - - %% CM Pool Sup - MFA = {?CM, start_link, [emqttd_stats:statsfun('clients/count', 'clients/max')]}, - PoolSup = emqttd_pool_sup:spec([?CM, hash, erlang:system_info(schedulers), MFA]), - - {ok, {{one_for_all, 10, 3600}, [PoolSup]}}. - -create_client_tab() -> - case ets:info(?TAB, name) of - undefined -> - ets:new(?TAB, [ordered_set, named_table, public, - {keypos, 2}, {write_concurrency, true}]); - _ -> - ok - end. - diff --git a/src/emqttd_config.erl b/src/emqttd_config.erl deleted file mode 100644 index 04d94b260..000000000 --- a/src/emqttd_config.erl +++ /dev/null @@ -1,114 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Hot Configuration -%% -%% TODO: How to persist the configuration? -%% -%% 1. Store in mnesia database? -%% 2. Store in dets? -%% 3. Store in data/app.config? -%% - --module(emqttd_config). - --export([read/1, write/2, dump/2, reload/1, get/2, get/3, set/3]). - --type(env() :: {atom(), term()}). - -%% @doc Read the configuration of an application. --spec(read(atom()) -> {ok, list(env())} | {error, term()}). -read(App) -> - %% TODO: - %% 1. Read the app.conf from etc folder - %% 2. Cuttlefish to read the conf - %% 3. Return the terms and schema - % {error, unsupported}. - {ok, read_(App)}. - -%% @doc Reload configuration of an application. --spec(reload(atom()) -> ok | {error, term()}). -reload(_App) -> - %% TODO - %% 1. Read the app.conf from etc folder - %% 2. Cuttlefish to generate config terms. - %% 3. set/3 to apply the config - ok. - --spec(write(atom(), list(env())) -> ok | {error, term()}). -write(App, Terms) -> - Configs = lists:map(fun({Key, Val}) -> - {cuttlefish_variable:tokenize(binary_to_list(Key)), binary_to_list(Val)} - end, Terms), - Path = lists:concat([code:priv_dir(App), "/", App, ".schema"]), - Schema = cuttlefish_schema:files([Path]), - case cuttlefish_generator:map(Schema, Configs) of - [{App, Configs1}] -> - emqttd_cli_config:write_config(App, Configs), - lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Configs1); - _ -> - error - end. - --spec(dump(atom(), list(env())) -> ok | {error, term()}). -dump(_App, _Terms) -> - %% TODO - ok. - --spec(set(atom(), list(), list()) -> ok). -set(App, Par, Val) -> - emqttd_cli_config:run(["config", - "set", - lists:concat([Par, "=", Val]), - lists:concat(["--app=", App])]). - --spec(get(atom(), list()) -> undefined | {ok, term()}). -get(App, Par) -> - case emqttd_cli_config:get_cfg(App, Par) of - undefined -> undefined; - Val -> {ok, Val} - end. - --spec(get(atom(), list(), atom()) -> term()). -get(App, Par, Def) -> - emqttd_cli_config:get_cfg(App, Par, Def). - - -read_(App) -> - Configs = emqttd_cli_config:read_config(App), - Path = lists:concat([code:priv_dir(App), "/", App, ".schema"]), - case filelib:is_file(Path) of - false -> - []; - true -> - {_, Mappings, _} = cuttlefish_schema:files([Path]), - OptionalCfg = lists:foldl(fun(Map, Acc) -> - Key = cuttlefish_mapping:variable(Map), - case proplists:get_value(Key, Configs) of - undefined -> - [{cuttlefish_variable:format(Key), "", cuttlefish_mapping:doc(Map), false} | Acc]; - _ -> Acc - end - end, [], Mappings), - RequiredCfg = lists:foldl(fun({Key, Val}, Acc) -> - case lists:keyfind(Key, 2, Mappings) of - false -> Acc; - Map -> - [{cuttlefish_variable:format(Key), Val, cuttlefish_mapping:doc(Map), true} | Acc] - end - end, [], Configs), - RequiredCfg ++ OptionalCfg - end. diff --git a/src/emqttd_ctl.erl b/src/emqttd_ctl.erl deleted file mode 100644 index a32a40172..000000000 --- a/src/emqttd_ctl.erl +++ /dev/null @@ -1,176 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_ctl). - --behaviour(gen_server). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_cli.hrl"). - --define(SERVER, ?MODULE). - -%% API Function Exports --export([start_link/0, register_cmd/2, register_cmd/3, unregister_cmd/1, - lookup/1, run/1]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {seq = 0}). - --define(CMD_TAB, mqttd_ctl_cmd). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -%% @doc Register a command --spec(register_cmd(atom(), {module(), atom()}) -> ok). -register_cmd(Cmd, MF) -> - register_cmd(Cmd, MF, []). - -%% @doc Register a command with opts --spec(register_cmd(atom(), {module(), atom()}, list()) -> ok). -register_cmd(Cmd, MF, Opts) -> - cast({register_cmd, Cmd, MF, Opts}). - -%% @doc Unregister a command --spec(unregister_cmd(atom()) -> ok). -unregister_cmd(Cmd) -> - cast({unregister_cmd, Cmd}). - -cast(Msg) -> gen_server:cast(?SERVER, Msg). - -%% @doc Run a command --spec(run([string()]) -> any()). -run([]) -> usage(), ok; - -run(["help"]) -> usage(), ok; - -run(["set"] = CmdS) when length(CmdS) =:= 1 -> - emqttd_cli_config:set_usage(), ok; - -run(["set" | _] = CmdS) -> - emqttd_cli_config:run(["config" | CmdS]), ok; - -run(["show" | _] = CmdS) -> - emqttd_cli_config:run(["config" | CmdS]), ok; - -run([CmdS|Args]) -> - case lookup(list_to_atom(CmdS)) of - [{Mod, Fun}] -> - try Mod:Fun(Args) of - _ -> ok - catch - _:Reason -> - io:format("Reason:~p, get_stacktrace:~p~n", - [Reason, erlang:get_stacktrace()]), - {error, Reason} - end; - [] -> - usage(), - {error, cmd_not_found} - end. - -%% @doc Lookup a command --spec(lookup(atom()) -> [{module(), atom()}]). -lookup(Cmd) -> - case ets:match(?CMD_TAB, {{'_', Cmd}, '$1', '_'}) of - [El] -> El; - [] -> [] - end. - -%% @doc Usage -usage() -> - ?PRINT("Usage: ~s~n", [?MODULE]), - [begin ?PRINT("~80..-s~n", [""]), Mod:Cmd(usage) end - || {_, {Mod, Cmd}, _} <- ets:tab2list(?CMD_TAB)]. - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - ets:new(?CMD_TAB, [ordered_set, named_table, protected]), - {ok, #state{seq = 0}}. - -handle_call(_Request, _From, State) -> - {reply, ok, State}. - -handle_cast({register_cmd, Cmd, MF, Opts}, State = #state{seq = Seq}) -> - case ets:match(?CMD_TAB, {{'$1', Cmd}, '_', '_'}) of - [] -> - ets:insert(?CMD_TAB, {{Seq, Cmd}, MF, Opts}); - [[OriginSeq] | _] -> - lager:warning("CLI: ~s is overidden by ~p", [Cmd, MF]), - ets:insert(?CMD_TAB, {{OriginSeq, Cmd}, MF, Opts}) - end, - noreply(next_seq(State)); - -handle_cast({unregister_cmd, Cmd}, State) -> - ets:match_delete(?CMD_TAB, {{'_', Cmd}, '_', '_'}), - noreply(State); - -handle_cast(_Msg, State) -> - noreply(State). - -handle_info(_Info, State) -> - noreply(State). - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal Function Definitions -%%-------------------------------------------------------------------- - -noreply(State) -> - {noreply, State, hibernate}. - -next_seq(State = #state{seq = Seq}) -> - State#state{seq = Seq + 1}. - --ifdef(TEST). - --include_lib("eunit/include/eunit.hrl"). - -register_cmd_test_() -> - {setup, - fun() -> - {ok, InitState} = emqttd_ctl:init([]), - InitState - end, - fun(State) -> - ok = emqttd_ctl:terminate(shutdown, State) - end, - fun(State = #state{seq = Seq}) -> - emqttd_ctl:handle_cast({register_cmd, test0, {?MODULE, test0}, []}, State), - [?_assertMatch([{{0,test0},{?MODULE, test0}, []}], ets:lookup(?CMD_TAB, {Seq,test0}))] - end - }. - --endif. diff --git a/src/emqttd_gc.erl b/src/emqttd_gc.erl deleted file mode 100644 index 6484a195d..000000000 --- a/src/emqttd_gc.erl +++ /dev/null @@ -1,50 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% GC Utility functions. - --module(emqttd_gc). - --author("Feng Lee "). - --export([conn_max_gc_count/0, reset_conn_gc_count/2, maybe_force_gc/2, - maybe_force_gc/3]). - --spec(conn_max_gc_count() -> integer()). -conn_max_gc_count() -> - case emqttd:env(conn_force_gc_count) of - {ok, I} when I > 0 -> I + rand:uniform(I); - {ok, I} when I =< 0 -> undefined; - undefined -> undefined - end. - --spec(reset_conn_gc_count(pos_integer(), tuple()) -> tuple()). -reset_conn_gc_count(Pos, State) -> - case element(Pos, State) of - undefined -> State; - _I -> setelement(Pos, State, conn_max_gc_count()) - end. - -maybe_force_gc(Pos, State) -> - maybe_force_gc(Pos, State, fun() -> ok end). -maybe_force_gc(Pos, State, Cb) -> - case element(Pos, State) of - undefined -> State; - I when I =< 0 -> Cb(), garbage_collect(), - reset_conn_gc_count(Pos, State); - I -> setelement(Pos, State, I - 1) - end. - diff --git a/src/emqttd_hooks.erl b/src/emqttd_hooks.erl deleted file mode 100644 index 4fc84f9e8..000000000 --- a/src/emqttd_hooks.erl +++ /dev/null @@ -1,189 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_hooks). - --behaviour(gen_server). - --author("Feng Lee "). - -%% Start --export([start_link/0]). - -%% Hooks API --export([add/3, add/4, delete/2, run/2, run/3, lookup/1]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {}). - --type(hooktag() :: atom() | string() | binary()). - --export_type([hooktag/0]). - --record(callback, {tag :: hooktag(), - function :: function(), - init_args = [] :: list(any()), - priority = 0 :: integer()}). - --record(hook, {name :: atom(), callbacks = [] :: list(#callback{})}). - --define(HOOK_TAB, mqtt_hook). - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -%%-------------------------------------------------------------------- -%% Hooks API -%%-------------------------------------------------------------------- - --spec(add(atom(), function() | {hooktag(), function()}, list(any())) -> ok). -add(HookPoint, Function, InitArgs) when is_function(Function) -> - add(HookPoint, {undefined, Function}, InitArgs, 0); - -add(HookPoint, {Tag, Function}, InitArgs) when is_function(Function) -> - add(HookPoint, {Tag, Function}, InitArgs, 0). - --spec(add(atom(), function() | {hooktag(), function()}, list(any()), integer()) -> ok). -add(HookPoint, Function, InitArgs, Priority) when is_function(Function) -> - add(HookPoint, {undefined, Function}, InitArgs, Priority); -add(HookPoint, {Tag, Function}, InitArgs, Priority) when is_function(Function) -> - gen_server:call(?MODULE, {add, HookPoint, {Tag, Function}, InitArgs, Priority}). - --spec(delete(atom(), function() | {hooktag(), function()}) -> ok). -delete(HookPoint, Function) when is_function(Function) -> - delete(HookPoint, {undefined, Function}); -delete(HookPoint, {Tag, Function}) when is_function(Function) -> - gen_server:call(?MODULE, {delete, HookPoint, {Tag, Function}}). - -%% @doc Run hooks without Acc. --spec(run(atom(), list(Arg :: any())) -> ok | stop). -run(HookPoint, Args) -> - run_(lookup(HookPoint), Args). - --spec(run(atom(), list(Arg :: any()), any()) -> any()). -run(HookPoint, Args, Acc) -> - run_(lookup(HookPoint), Args, Acc). - -%% @private -run_([#callback{function = Fun, init_args = InitArgs} | Callbacks], Args) -> - case apply(Fun, lists:append([Args, InitArgs])) of - ok -> run_(Callbacks, Args); - stop -> stop; - _Any -> run_(Callbacks, Args) - end; - -run_([], _Args) -> - ok. - -%% @private -run_([#callback{function = Fun, init_args = InitArgs} | Callbacks], Args, Acc) -> - case apply(Fun, lists:append([Args, [Acc], InitArgs])) of - ok -> run_(Callbacks, Args, Acc); - {ok, NewAcc} -> run_(Callbacks, Args, NewAcc); - stop -> {stop, Acc}; - {stop, NewAcc} -> {stop, NewAcc}; - _Any -> run_(Callbacks, Args, Acc) - end; - -run_([], _Args, Acc) -> - {ok, Acc}. - --spec(lookup(atom()) -> [#callback{}]). -lookup(HookPoint) -> - case ets:lookup(?HOOK_TAB, HookPoint) of - [#hook{callbacks = Callbacks}] -> Callbacks; - [] -> [] - end. - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([]) -> - ets:new(?HOOK_TAB, [set, protected, named_table, {keypos, #hook.name}]), - {ok, #state{}}. - -handle_call({add, HookPoint, {Tag, Function}, InitArgs, Priority}, _From, State) -> - Callback = #callback{tag = Tag, function = Function, - init_args = InitArgs, priority = Priority}, - {reply, - case ets:lookup(?HOOK_TAB, HookPoint) of - [#hook{callbacks = Callbacks}] -> - case contain_(Tag, Function, Callbacks) of - false -> - insert_hook_(HookPoint, add_callback_(Callback, Callbacks)); - true -> - {error, already_hooked} - end; - [] -> - insert_hook_(HookPoint, [Callback]) - end, State}; - -handle_call({delete, HookPoint, {Tag, Function}}, _From, State) -> - {reply, - case ets:lookup(?HOOK_TAB, HookPoint) of - [#hook{callbacks = Callbacks}] -> - case contain_(Tag, Function, Callbacks) of - true -> - insert_hook_(HookPoint, del_callback_(Tag, Function, Callbacks)); - false -> - {error, not_found} - end; - [] -> - {error, not_found} - end, State}; - -handle_call(Req, _From, State) -> - {reply, {error, {unexpected_request, Req}}, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -insert_hook_(HookPoint, Callbacks) -> - ets:insert(?HOOK_TAB, #hook{name = HookPoint, callbacks = Callbacks}), ok. - -add_callback_(Callback, Callbacks) -> - lists:keymerge(#callback.priority, Callbacks, [Callback]). - -del_callback_(Tag, Function, Callbacks) -> - lists:filter( - fun(#callback{tag = Tag1, function = Func1}) -> - not ((Tag =:= Tag1) andalso (Function =:= Func1)) - end, Callbacks). - -contain_(_Tag, _Function, []) -> - false; -contain_(Tag, Function, [#callback{tag = Tag, function = Function}|_Callbacks]) -> - true; -contain_(Tag, Function, [_Callback | Callbacks]) -> - contain_(Tag, Function, Callbacks). - diff --git a/src/emqttd_http.erl b/src/emqttd_http.erl deleted file mode 100644 index c229f4de2..000000000 --- a/src/emqttd_http.erl +++ /dev/null @@ -1,235 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc HTTP publish API and websocket client. - --module(emqttd_http). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --import(proplists, [get_value/2, get_value/3]). - --export([http_handler/0, handle_request/2, http_api/0, inner_handle_request/2]). - --include("emqttd_internal.hrl"). - --record(state, {dispatch}). - -http_handler() -> - APIs = http_api(), - State = #state{dispatch = dispatcher(APIs)}, - {?MODULE, handle_request, [State]}. - -http_api() -> - Attr = emqttd_rest_api:module_info(attributes), - [{Regexp, Method, Function, Args} || {http_api, [{Regexp, Method, Function, Args}]} <- Attr]. - -%%-------------------------------------------------------------------- -%% Handle HTTP Request -%%-------------------------------------------------------------------- -handle_request(Req, State) -> - {Path, _, _} = mochiweb_util:urlsplit_path(Req:get(raw_path)), - case Path of - "/status" -> - handle_request("/status", Req, Req:get(method)); - "/" -> - handle_request("/", Req, Req:get(method)); - "/api/v2/auth" -> - handle_request(Path, Req, State); - _ -> - if_authorized(Req, fun() -> handle_request(Path, Req, State) end) - end. - -inner_handle_request(Req, State) -> - {Path, _, _} = mochiweb_util:urlsplit_path(Req:get(raw_path)), - case Path of - "/api/v2/auth" -> handle_request(Path, Req, State); - _ -> if_authorized(Req, fun() -> handle_request(Path, Req, State) end) - end. - -handle_request("/api/v2/" ++ Url, Req, #state{dispatch = Dispatch}) -> - Dispatch(Req, Url); - -handle_request("/status", Req, Method) when Method =:= 'HEAD'; Method =:= 'GET' -> - {InternalStatus, _ProvidedStatus} = init:get_status(), - AppStatus = case lists:keysearch(emqttd, 1, application:which_applications()) of - false -> not_running; - {value, _Val} -> running - end, - Status = io_lib:format("Node ~s is ~s~nemqttd is ~s", - [node(), InternalStatus, AppStatus]), - Req:ok({"text/plain", iolist_to_binary(Status)}); - -handle_request("/", Req, Method) when Method =:= 'HEAD'; Method =:= 'GET' -> - respond(Req, 200, api_list()); - -handle_request(_, Req, #state{}) -> - respond(Req, 404, []). - -dispatcher(APIs) -> - fun(Req, Url) -> - Method = Req:get(method), - case filter(APIs, Url, Method) of - [{Regexp, _Method, Function, FilterArgs}] -> - case params(Req) of - {error, Error1} -> - respond(Req, 200, Error1); - Params -> - case {check_params(Params, FilterArgs), - check_params_type(Params, FilterArgs)} of - {true, true} -> - {match, [MatchList0]} = re:run(Url, Regexp, [global, {capture, all_but_first, list}]), - MatchList = lists:map(fun mochiweb_util:unquote/1, MatchList0), - Args = lists:append([[Method, Params], MatchList]), - lager:debug("Mod:~p, Fun:~p, Args:~p", [emqttd_rest_api, Function, Args]), - case catch apply(emqttd_rest_api, Function, Args) of - {ok, Data} -> - respond(Req, 200, [{code, ?SUCCESS}, {result, Data}]); - {error, Error} -> - respond(Req, 200, Error); - {'EXIT', Reason} -> - lager:error("Execute API '~s' Error: ~p", [Url, Reason]), - respond(Req, 404, []) - end; - {false, _} -> - respond(Req, 200, [{code, ?ERROR7}, {message, <<"params error">>}]); - {_, false} -> - respond(Req, 200, [{code, ?ERROR8}, {message, <<"params type error">>}]) - end - end; - _ -> - lager:error("No match Url:~p", [Url]), - respond(Req, 404, []) - end - end. - -% %%-------------------------------------------------------------------- -% %% Basic Authorization -% %%-------------------------------------------------------------------- -if_authorized(Req, Fun) -> - case authorized(Req) of - true -> Fun(); - false -> respond(Req, 401, []) - end. - -authorized(Req) -> - case Req:get_header_value("Authorization") of - undefined -> - false; - "Basic " ++ BasicAuth -> - {Username, Password} = user_passwd(BasicAuth), - case emqttd_mgmt:check_user(Username, Password) of - ok -> - true; - {error, Reason} -> - lager:error("HTTP Auth failure: username=~s, reason=~p", [Username, Reason]), - false - end - end. - -user_passwd(BasicAuth) -> - list_to_tuple(binary:split(base64:decode(BasicAuth), <<":">>)). - -respond(Req, 401, Data) -> - Req:respond({401, [{"WWW-Authenticate", "Basic Realm=\"emqx control center\""}], Data}); -respond(Req, 404, Data) -> - Req:respond({404, [{"Content-Type", "text/plain"}], Data}); -respond(Req, 200, Data) -> - Req:respond({200, [{"Content-Type", "application/json"}], to_json(Data)}); -respond(Req, Code, Data) -> - Req:respond({Code, [{"Content-Type", "text/plain"}], Data}). - -filter(APIs, Url, Method) -> - lists:filter(fun({Regexp, Method1, _Function, _Args}) -> - case re:run(Url, Regexp, [global, {capture, all_but_first, list}]) of - {match, _} -> Method =:= Method1; - _ -> false - end - end, APIs). - -params(Req) -> - Method = Req:get(method), - case Method of - 'GET' -> - mochiweb_request:parse_qs(Req); - _ -> - case Req:recv_body() of - <<>> -> []; - undefined -> []; - Body -> - case jsx:is_json(Body) of - true -> jsx:decode(Body); - false -> - lager:error("Body:~p", [Body]), - {error, [{code, ?ERROR9}, {message, <<"Body not json">>}]} - end - end - end. - -check_params(_Params, Args) when Args =:= [] -> - true; -check_params(Params, Args)-> - not lists:any(fun({Item, _Type}) -> undefined =:= proplists:get_value(Item, Params) end, Args). - -check_params_type(_Params, Args) when Args =:= [] -> - true; -check_params_type(Params, Args) -> - not lists:any(fun({Item, Type}) -> - Val = proplists:get_value(Item, Params), - case Type of - int -> not is_integer(Val); - binary -> not is_binary(Val); - bool -> not is_boolean(Val) - end - end, Args). - -to_json([]) -> <<"[]">>; -to_json(Data) -> iolist_to_binary(mochijson2:encode(Data)). - -api_list() -> - [{paths, [<<"api/v2/management/nodes">>, - <<"api/v2/management/nodes/{node_name}">>, - <<"api/v2/monitoring/nodes">>, - <<"api/v2/monitoring/nodes/{node_name}">>, - <<"api/v2/monitoring/listeners">>, - <<"api/v2/monitoring/listeners/{node_name}">>, - <<"api/v2/monitoring/metrics/">>, - <<"api/v2/monitoring/metrics/{node_name}">>, - <<"api/v2/monitoring/stats">>, - <<"api/v2/monitoring/stats/{node_name}">>, - <<"api/v2/nodes/{node_name}/clients">>, - <<"api/v2/nodes/{node_name}/clients/{clientid}">>, - <<"api/v2/clients/{clientid}">>, - <<"api/v2/clients/{clientid}/clean_acl_cache">>, - <<"api/v2/nodes/{node_name}/sessions">>, - <<"api/v2/nodes/{node_name}/sessions/{clientid}">>, - <<"api/v2/sessions/{clientid}">>, - <<"api/v2/nodes/{node_name}/subscriptions">>, - <<"api/v2/nodes/{node_name}/subscriptions/{clientid}">>, - <<"api/v2/subscriptions/{clientid}">>, - <<"api/v2/routes">>, - <<"api/v2/routes/{topic}">>, - <<"api/v2/mqtt/publish">>, - <<"api/v2/mqtt/subscribe">>, - <<"api/v2/mqtt/unsubscribe">>, - <<"api/v2/nodes/{node_name}/plugins">>, - <<"api/v2/nodes/{node_name}/plugins/{plugin_name}">>, - <<"api/v2/configs/{app}">>, - <<"api/v2/nodes/{node_name}/configs/{app}">>]}]. diff --git a/src/emqttd_inflight.erl b/src/emqttd_inflight.erl deleted file mode 100644 index be7517197..000000000 --- a/src/emqttd_inflight.erl +++ /dev/null @@ -1,94 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Inflight Window that wraps the gb_trees. - --module(emqttd_inflight). - --author("Feng Lee "). - --export([new/1, contain/2, lookup/2, insert/3, update/3, delete/2, values/1, - to_list/1, size/1, max_size/1, is_full/1, is_empty/1, window/1]). - --type(inflight() :: {?MODULE, list()}). - --export_type([inflight/0]). - --spec(new(non_neg_integer()) -> inflight()). -new(MaxSize) when MaxSize >= 0 -> - {?MODULE, [MaxSize, gb_trees:empty()]}. - --spec(contain(Key :: any(), inflight()) -> boolean()). -contain(Key, {?MODULE, [_MaxSize, Tree]}) -> - gb_trees:is_defined(Key, Tree). - --spec(lookup(Key :: any(), inflight()) -> any()). -lookup(Key, {?MODULE, [_MaxSize, Tree]}) -> - gb_trees:get(Key, Tree). - --spec(insert(Key :: any(), Value :: any(), inflight()) -> inflight()). -insert(Key, Value, {?MODULE, [MaxSize, Tree]}) -> - {?MODULE, [MaxSize, gb_trees:insert(Key, Value, Tree)]}. - --spec(delete(Key :: any(), inflight()) -> inflight()). -delete(Key, {?MODULE, [MaxSize, Tree]}) -> - {?MODULE, [MaxSize, gb_trees:delete(Key, Tree)]}. - --spec(update(Key :: any(), Val :: any(), inflight()) -> inflight()). -update(Key, Val, {?MODULE, [MaxSize, Tree]}) -> - {?MODULE, [MaxSize, gb_trees:update(Key, Val, Tree)]}. - --spec(is_full(inflight()) -> boolean()). -is_full({?MODULE, [0, _Tree]}) -> - false; -is_full({?MODULE, [MaxSize, Tree]}) -> - MaxSize =< gb_trees:size(Tree). - --spec(is_empty(inflight()) -> boolean()). -is_empty({?MODULE, [_MaxSize, Tree]}) -> - gb_trees:is_empty(Tree). - --spec(smallest(inflight()) -> {K :: any(), V :: any()}). -smallest({?MODULE, [_MaxSize, Tree]}) -> - gb_trees:smallest(Tree). - --spec(largest(inflight()) -> {K :: any(), V :: any()}). -largest({?MODULE, [_MaxSize, Tree]}) -> - gb_trees:largest(Tree). - --spec(values(inflight()) -> list()). -values({?MODULE, [_MaxSize, Tree]}) -> - gb_trees:values(Tree). - --spec(to_list(inflight()) -> list({K :: any(), V :: any()})). -to_list({?MODULE, [_MaxSize, Tree]}) -> - gb_trees:to_list(Tree). - --spec(window(inflight()) -> list()). -window(Inflight = {?MODULE, [_MaxSize, Tree]}) -> - case gb_trees:is_empty(Tree) of - true -> []; - false -> [Key || {Key, _Val} <- [smallest(Inflight), largest(Inflight)]] - end. - --spec(size(inflight()) -> non_neg_integer()). -size({?MODULE, [_MaxSize, Tree]}) -> - gb_trees:size(Tree). - --spec(max_size(inflight()) -> non_neg_integer()). -max_size({?MODULE, [MaxSize, _Tree]}) -> - MaxSize. - diff --git a/src/emqttd_message.erl b/src/emqttd_message.erl deleted file mode 100644 index 86918e47a..000000000 --- a/src/emqttd_message.erl +++ /dev/null @@ -1,158 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc MQTT Message Functions - --module(emqttd_message). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --export([make/3, make/4, from_packet/1, from_packet/2, from_packet/3, - to_packet/1]). - --export([set_flag/1, set_flag/2, unset_flag/1, unset_flag/2]). - --export([format/1]). - --type(msg_from() :: atom() | {binary(), undefined | binary()}). - -%% @doc Make a message --spec(make(msg_from(), binary(), binary()) -> mqtt_message()). -make(From, Topic, Payload) -> - make(From, ?QOS_0, Topic, Payload). - --spec(make(msg_from(), mqtt_qos(), binary(), binary()) -> mqtt_message()). -make(From, Qos, Topic, Payload) -> - #mqtt_message{id = msgid(), - from = From, - qos = ?QOS_I(Qos), - topic = Topic, - payload = Payload, - timestamp = os:timestamp()}. - -%% @doc Message from Packet --spec(from_packet(mqtt_packet()) -> mqtt_message()). -from_packet(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, - retain = Retain, - qos = Qos, - dup = Dup}, - variable = #mqtt_packet_publish{topic_name = Topic, - packet_id = PacketId}, - payload = Payload}) -> - #mqtt_message{id = msgid(), - pktid = PacketId, - qos = Qos, - retain = Retain, - dup = Dup, - topic = Topic, - payload = Payload, - timestamp = os:timestamp()}; - -from_packet(#mqtt_packet_connect{will_flag = false}) -> - undefined; - -from_packet(#mqtt_packet_connect{client_id = ClientId, - username = Username, - will_retain = Retain, - will_qos = Qos, - will_topic = Topic, - will_msg = Msg}) -> - #mqtt_message{id = msgid(), - topic = Topic, - from = {ClientId, Username}, - retain = Retain, - qos = Qos, - dup = false, - payload = Msg, - timestamp = os:timestamp()}. - -from_packet(ClientId, Packet) -> - Msg = from_packet(Packet), - Msg#mqtt_message{from = ClientId}. - -from_packet(Username, ClientId, Packet) -> - Msg = from_packet(Packet), - Msg#mqtt_message{from = {ClientId, Username}}. - -msgid() -> emqttd_guid:gen(). - -%% @doc Message to packet --spec(to_packet(mqtt_message()) -> mqtt_packet()). -to_packet(#mqtt_message{pktid = PkgId, - qos = Qos, - retain = Retain, - dup = Dup, - topic = Topic, - payload = Payload}) -> - - #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, - qos = Qos, - retain = Retain, - dup = Dup}, - variable = #mqtt_packet_publish{topic_name = Topic, - packet_id = if - Qos =:= ?QOS_0 -> undefined; - true -> PkgId - end - }, - payload = Payload}. - -%% @doc set dup, retain flag --spec(set_flag(mqtt_message()) -> mqtt_message()). -set_flag(Msg) -> - Msg#mqtt_message{dup = true, retain = true}. - --spec(set_flag(atom(), mqtt_message()) -> mqtt_message()). -set_flag(dup, Msg = #mqtt_message{dup = false}) -> - Msg#mqtt_message{dup = true}; -set_flag(sys, Msg = #mqtt_message{sys = false}) -> - Msg#mqtt_message{sys = true}; -set_flag(retain, Msg = #mqtt_message{retain = false}) -> - Msg#mqtt_message{retain = true}; -set_flag(Flag, Msg) when Flag =:= dup orelse Flag =:= retain -> Msg. - -%% @doc Unset dup, retain flag --spec(unset_flag(mqtt_message()) -> mqtt_message()). -unset_flag(Msg) -> - Msg#mqtt_message{dup = false, retain = false}. - --spec(unset_flag(dup | retain | atom(), mqtt_message()) -> mqtt_message()). -unset_flag(dup, Msg = #mqtt_message{dup = true}) -> - Msg#mqtt_message{dup = false}; -unset_flag(retain, Msg = #mqtt_message{retain = true}) -> - Msg#mqtt_message{retain = false}; -unset_flag(Flag, Msg) when Flag =:= dup orelse Flag =:= retain -> Msg. - -%% @doc Format MQTT Message -format(#mqtt_message{id = MsgId, pktid = PktId, from = {ClientId, Username}, - qos = Qos, retain = Retain, dup = Dup, topic =Topic}) -> - io_lib:format("Message(Q~p, R~p, D~p, MsgId=~p, PktId=~p, From=~s/~s, Topic=~s)", - [i(Qos), i(Retain), i(Dup), MsgId, PktId, Username, ClientId, Topic]); - -%% TODO:... -format(#mqtt_message{id = MsgId, pktid = PktId, from = From, - qos = Qos, retain = Retain, dup = Dup, topic =Topic}) -> - io_lib:format("Message(Q~p, R~p, D~p, MsgId=~p, PktId=~p, From=~s, Topic=~s)", - [i(Qos), i(Retain), i(Dup), MsgId, PktId, From, Topic]). - -i(true) -> 1; -i(false) -> 0; -i(I) when is_integer(I) -> I. - diff --git a/src/emqttd_metrics.erl b/src/emqttd_metrics.erl deleted file mode 100644 index 37d897b67..000000000 --- a/src/emqttd_metrics.erl +++ /dev/null @@ -1,295 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_metrics). - --behaviour(gen_server). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --define(SERVER, ?MODULE). - -%% API Function Exports --export([start_link/0]). - -%% Received/Sent Metrics --export([received/1, sent/1]). - --export([all/0, value/1, inc/1, inc/2, inc/3, dec/2, dec/3, set/2]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {tick}). - --define(METRIC_TAB, mqtt_metric). - -%% Bytes sent and received of Broker --define(SYSTOP_BYTES, [ - {counter, 'bytes/received'}, % Total bytes received - {counter, 'bytes/sent'} % Total bytes sent -]). - -%% Packets sent and received of Broker --define(SYSTOP_PACKETS, [ - {counter, 'packets/received'}, % All Packets received - {counter, 'packets/sent'}, % All Packets sent - {counter, 'packets/connect'}, % CONNECT Packets received - {counter, 'packets/connack'}, % CONNACK Packets sent - {counter, 'packets/publish/received'}, % PUBLISH packets received - {counter, 'packets/publish/sent'}, % PUBLISH packets sent - {counter, 'packets/puback/received'}, % PUBACK packets received - {counter, 'packets/puback/sent'}, % PUBACK packets sent - {counter, 'packets/puback/missed'}, % PUBACK packets missed - {counter, 'packets/pubrec/received'}, % PUBREC packets received - {counter, 'packets/pubrec/sent'}, % PUBREC packets sent - {counter, 'packets/pubrec/missed'}, % PUBREC packets missed - {counter, 'packets/pubrel/received'}, % PUBREL packets received - {counter, 'packets/pubrel/sent'}, % PUBREL packets sent - {counter, 'packets/pubrel/missed'}, % PUBREL packets missed - {counter, 'packets/pubcomp/received'}, % PUBCOMP packets received - {counter, 'packets/pubcomp/sent'}, % PUBCOMP packets sent - {counter, 'packets/pubcomp/missed'}, % PUBCOMP packets missed - {counter, 'packets/subscribe'}, % SUBSCRIBE Packets received - {counter, 'packets/suback'}, % SUBACK packets sent - {counter, 'packets/unsubscribe'}, % UNSUBSCRIBE Packets received - {counter, 'packets/unsuback'}, % UNSUBACK Packets sent - {counter, 'packets/pingreq'}, % PINGREQ packets received - {counter, 'packets/pingresp'}, % PINGRESP Packets sent - {counter, 'packets/disconnect'} % DISCONNECT Packets received -]). - -%% Messages sent and received of broker --define(SYSTOP_MESSAGES, [ - {counter, 'messages/received'}, % All Messages received - {counter, 'messages/sent'}, % All Messages sent - {counter, 'messages/qos0/received'}, % QoS0 Messages received - {counter, 'messages/qos0/sent'}, % QoS0 Messages sent - {counter, 'messages/qos1/received'}, % QoS1 Messages received - {counter, 'messages/qos1/sent'}, % QoS1 Messages sent - {counter, 'messages/qos2/received'}, % QoS2 Messages received - {counter, 'messages/qos2/sent'}, % QoS2 Messages sent - {counter, 'messages/qos2/dropped'}, % QoS2 Messages dropped - {gauge, 'messages/retained'}, % Messagea retained - {counter, 'messages/dropped'} % Messages dropped -]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -%% @doc Start the metrics server --spec(start_link() -> {ok, pid()} | ignore | {error, term()}). -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -%% @doc Count packets received. --spec(received(mqtt_packet()) -> ignore | non_neg_integer()). -received(Packet) -> - inc('packets/received'), - received1(Packet). -received1(?PUBLISH_PACKET(Qos, _PktId)) -> - inc('packets/publish/received'), - inc('messages/received'), - qos_received(Qos); -received1(?PACKET(Type)) -> - received2(Type). -received2(?CONNECT) -> - inc('packets/connect'); -received2(?PUBACK) -> - inc('packets/puback/received'); -received2(?PUBREC) -> - inc('packets/pubrec/received'); -received2(?PUBREL) -> - inc('packets/pubrel/received'); -received2(?PUBCOMP) -> - inc('packets/pubcomp/received'); -received2(?SUBSCRIBE) -> - inc('packets/subscribe'); -received2(?UNSUBSCRIBE) -> - inc('packets/unsubscribe'); -received2(?PINGREQ) -> - inc('packets/pingreq'); -received2(?DISCONNECT) -> - inc('packets/disconnect'); -received2(_) -> - ignore. -qos_received(?QOS_0) -> - inc('messages/qos0/received'); -qos_received(?QOS_1) -> - inc('messages/qos1/received'); -qos_received(?QOS_2) -> - inc('messages/qos2/received'). - -%% @doc Count packets received. Will not count $SYS PUBLISH. --spec(sent(mqtt_packet()) -> ignore | non_neg_integer()). -sent(?PUBLISH_PACKET(_Qos, <<"$SYS/", _/binary>>, _, _)) -> - ignore; -sent(Packet) -> - inc('packets/sent'), - sent1(Packet). -sent1(?PUBLISH_PACKET(Qos, _PktId)) -> - inc('packets/publish/sent'), - inc('messages/sent'), - qos_sent(Qos); -sent1(?PACKET(Type)) -> - sent2(Type). -sent2(?CONNACK) -> - inc('packets/connack'); -sent2(?PUBACK) -> - inc('packets/puback/sent'); -sent2(?PUBREC) -> - inc('packets/pubrec/sent'); -sent2(?PUBREL) -> - inc('packets/pubrel/sent'); -sent2(?PUBCOMP) -> - inc('packets/pubcomp/sent'); -sent2(?SUBACK) -> - inc('packets/suback'); -sent2(?UNSUBACK) -> - inc('packets/unsuback'); -sent2(?PINGRESP) -> - inc('packets/pingresp'); -sent2(_Type) -> - ignore. -qos_sent(?QOS_0) -> - inc('messages/qos0/sent'); -qos_sent(?QOS_1) -> - inc('messages/qos1/sent'); -qos_sent(?QOS_2) -> - inc('messages/qos2/sent'). - -%% @doc Get all metrics --spec(all() -> [{atom(), non_neg_integer()}]). -all() -> - maps:to_list( - ets:foldl( - fun({{Metric, _N}, Val}, Map) -> - case maps:find(Metric, Map) of - {ok, Count} -> maps:put(Metric, Count+Val, Map); - error -> maps:put(Metric, Val, Map) - end - end, #{}, ?METRIC_TAB)). - -%% @doc Get metric value --spec(value(atom()) -> non_neg_integer()). -value(Metric) -> - lists:sum(ets:select(?METRIC_TAB, [{{{Metric, '_'}, '$1'}, [], ['$1']}])). - -%% @doc Increase counter --spec(inc(atom()) -> non_neg_integer()). -inc(Metric) -> - inc(counter, Metric, 1). - -%% @doc Increase metric value --spec(inc({counter | gauge, atom()} | atom(), pos_integer()) -> non_neg_integer()). -inc({gauge, Metric}, Val) -> - inc(gauge, Metric, Val); -inc({counter, Metric}, Val) -> - inc(counter, Metric, Val); -inc(Metric, Val) when is_atom(Metric) -> - inc(counter, Metric, Val). - -%% @doc Increase metric value --spec(inc(counter | gauge, atom(), pos_integer()) -> pos_integer()). -inc(gauge, Metric, Val) -> - ets:update_counter(?METRIC_TAB, key(gauge, Metric), {2, Val}); -inc(counter, Metric, Val) -> - ets:update_counter(?METRIC_TAB, key(counter, Metric), {2, Val}). - -%% @doc Decrease metric value --spec(dec(gauge, atom()) -> integer()). -dec(gauge, Metric) -> - dec(gauge, Metric, 1). - -%% @doc Decrease metric value --spec(dec(gauge, atom(), pos_integer()) -> integer()). -dec(gauge, Metric, Val) -> - ets:update_counter(?METRIC_TAB, key(gauge, Metric), {2, -Val}). - -%% @doc Set metric value -set(Metric, Val) when is_atom(Metric) -> - set(gauge, Metric, Val). -set(gauge, Metric, Val) -> - ets:insert(?METRIC_TAB, {key(gauge, Metric), Val}). - -%% @doc Metric Key -key(gauge, Metric) -> - {Metric, 0}; -key(counter, Metric) -> - {Metric, erlang:system_info(scheduler_id)}. - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - emqttd_time:seed(), - Metrics = ?SYSTOP_BYTES ++ ?SYSTOP_PACKETS ++ ?SYSTOP_MESSAGES, - % Create metrics table - ets:new(?METRIC_TAB, [set, public, named_table, {write_concurrency, true}]), - % Init metrics - [create_metric(Metric) || Metric <- Metrics], - % $SYS Topics for metrics - % [ok = emqttd:create(topic, metric_topic(Topic)) || {_, Topic} <- Metrics], - % Tick to publish metrics - {ok, #state{tick = emqttd_broker:start_tick(tick)}, hibernate}. - -handle_call(_Req, _From, State) -> - {reply, error, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(tick, State) -> - % publish metric message - [publish(Metric, Val) || {Metric, Val} <- all()], - {noreply, State, hibernate}; - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, #state{tick = TRef}) -> - emqttd_broker:stop_tick(TRef). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -publish(Metric, Val) -> - Msg = emqttd_message:make(metrics, metric_topic(Metric), bin(Val)), - emqttd:publish(emqttd_message:set_flag(sys, Msg)). - -create_metric({gauge, Name}) -> - ets:insert(?METRIC_TAB, {{Name, 0}, 0}); - -create_metric({counter, Name}) -> - Schedulers = lists:seq(1, erlang:system_info(schedulers)), - ets:insert(?METRIC_TAB, [{{Name, I}, 0} || I <- Schedulers]). - -metric_topic(Metric) -> - emqttd_topic:systop(list_to_binary(lists:concat(['metrics/', Metric]))). - -bin(I) when is_integer(I) -> list_to_binary(integer_to_list(I)). - diff --git a/src/emqttd_mgmt.erl b/src/emqttd_mgmt.erl deleted file mode 100644 index 2052d68fc..000000000 --- a/src/emqttd_mgmt.erl +++ /dev/null @@ -1,505 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_mgmt). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --include("emqttd_internal.hrl"). - --include_lib("stdlib/include/qlc.hrl"). - --record(mqtt_admin, {username, password, tags}). - --define(EMPTY_KEY(Key), ((Key == undefined) orelse (Key == <<>>))). - --import(proplists, [get_value/2]). - --export([brokers/0, broker/1, metrics/0, metrics/1, stats/1, stats/0, - plugins/0, plugins/1, listeners/0, listener/1, nodes_info/0, node_info/1]). - --export([plugin_list/1, plugin_unload/2, plugin_load/2]). - --export([client_list/4, session_list/4, route_list/3, subscription_list/4, alarm_list/0]). - --export([client/1, session/1, route/1, subscription/1]). - --export([query_table/4, lookup_table/3]). - --export([publish/1, subscribe/1, unsubscribe/1]). - --export([kick_client/1, kick_client/2, clean_acl_cache/2, clean_acl_cache/3]). - --export([modify_config/2, modify_config/3, modify_config/4, get_configs/0, get_config/1, - get_plugin_config/1, get_plugin_config/2, modify_plugin_config/2, modify_plugin_config/3]). - --export([add_user/3, check_user/2, user_list/0, lookup_user/1, - update_user/2, change_password/3, remove_user/1]). - --define(KB, 1024). --define(MB, (1024*1024)). --define(GB, (1024*1024*1024)). - -brokers() -> - [{Node, broker(Node)} || Node <- ekka_mnesia:running_nodes()]. - -broker(Node) when Node =:= node() -> - emqttd_broker:info(); -broker(Node) -> - rpc_call(Node, broker, [Node]). - -metrics() -> - [{Node, metrics(Node)} || Node <- ekka_mnesia:running_nodes()]. - -metrics(Node) when Node =:= node() -> - emqttd_metrics:all(); -metrics(Node) -> - rpc_call(Node, metrics, [Node]). - -stats() -> - [{Node, stats(Node)} || Node <- ekka_mnesia:running_nodes()]. - -stats(Node) when Node =:= node() -> - emqttd_stats:getstats(); -stats(Node) -> - rpc_call(Node, stats, [Node]). - -plugins() -> - [{Node, plugins(Node)} || Node <- ekka_mnesia:running_nodes()]. - -plugins(Node) when Node =:= node() -> - emqttd_plugins:list(Node); -plugins(Node) -> - rpc_call(Node, plugins, [Node]). - -listeners() -> - [{Node, listener(Node)} || Node <- ekka_mnesia:running_nodes()]. - -listener(Node) when Node =:= node() -> - lists:map(fun({{Protocol, ListenOn}, Pid}) -> - Info = [{acceptors, esockd:get_acceptors(Pid)}, - {max_clients, esockd:get_max_clients(Pid)}, - {current_clients,esockd:get_current_clients(Pid)}, - {shutdown_count, esockd:get_shutdown_count(Pid)}], - {Protocol, ListenOn, Info} - end, esockd:listeners()); - -listener(Node) -> - rpc_call(Node, listener, [Node]). - -nodes_info() -> - Running = mnesia:system_info(running_db_nodes), - Stopped = mnesia:system_info(db_nodes) -- Running, - DownNodes = lists:map(fun stop_node/1, Stopped), - [node_info(Node) || Node <- Running] ++ DownNodes. - -node_info(Node) when Node =:= node() -> - CpuInfo = [{K, list_to_binary(V)} || {K, V} <- emqttd_vm:loads()], - Memory = emqttd_vm:get_memory(), - OtpRel = "R" ++ erlang:system_info(otp_release) ++ "/" ++ erlang:system_info(version), - [{name, node()}, - {otp_release, list_to_binary(OtpRel)}, - {memory_total, kmg(get_value(allocated, Memory))}, - {memory_used, kmg(get_value(used, Memory))}, - {process_available, erlang:system_info(process_limit)}, - {process_used, erlang:system_info(process_count)}, - {max_fds, get_value(max_fds, erlang:system_info(check_io))}, - {clients, ets:info(mqtt_client, size)}, - {node_status, 'Running'} | CpuInfo]; - -node_info(Node) -> - rpc_call(Node, node_info, [Node]). - -stop_node(Node) -> - [{name, Node}, {node_status, 'Stopped'}]. -%%-------------------------------------------------------- -%% plugins -%%-------------------------------------------------------- -plugin_list(Node) when Node =:= node() -> - emqttd_plugins:list(); -plugin_list(Node) -> - rpc_call(Node, plugin_list, [Node]). - -plugin_load(Node, PluginName) when Node =:= node() -> - emqttd_plugins:load(PluginName); -plugin_load(Node, PluginName) -> - rpc_call(Node, plugin_load, [Node, PluginName]). - -plugin_unload(Node, PluginName) when Node =:= node() -> - emqttd_plugins:unload(PluginName); -plugin_unload(Node, PluginName) -> - rpc_call(Node, plugin_unload, [Node, PluginName]). - -%%-------------------------------------------------------- -%% client -%%-------------------------------------------------------- -client_list(Node, Key, PageNo, PageSize) when Node =:= node() -> - client_list(Key, PageNo, PageSize); -client_list(Node, Key, PageNo, PageSize) -> - rpc_call(Node, client_list, [Node, Key, PageNo, PageSize]). - -client(ClientId) -> - lists:flatten([client_list(Node, ClientId, 1, 20) || Node <- ekka_mnesia:running_nodes()]). - -%%-------------------------------------------------------- -%% session -%%-------------------------------------------------------- -session_list(Node, Key, PageNo, PageSize) when Node =:= node() -> - session_list(Key, PageNo, PageSize); -session_list(Node, Key, PageNo, PageSize) -> - rpc_call(Node, session_list, [Node, Key, PageNo, PageSize]). - -session(ClientId) -> - lists:flatten([session_list(Node, ClientId, 1, 20) || Node <- ekka_mnesia:running_nodes()]). - -%%-------------------------------------------------------- -%% subscription -%%-------------------------------------------------------- -subscription_list(Node, Key, PageNo, PageSize) when Node =:= node() -> - subscription_list(Key, PageNo, PageSize); -subscription_list(Node, Key, PageNo, PageSize) -> - rpc_call(Node, subscription_list, [Node, Key, PageNo, PageSize]). - -subscription(Key) -> - lists:flatten([subscription_list(Node, Key, 1, 20) || Node <- ekka_mnesia:running_nodes()]). - -%%-------------------------------------------------------- -%% Routes -%%-------------------------------------------------------- -route(Key) -> route_list(Key, 1, 20). - -%%-------------------------------------------------------- -%% alarm -%%-------------------------------------------------------- -alarm_list() -> - emqttd_alarm:get_alarms(). - -query_table(Qh, PageNo, PageSize, TotalNum) -> - Cursor = qlc:cursor(Qh), - case PageNo > 1 of - true -> qlc:next_answers(Cursor, (PageNo - 1) * PageSize); - false -> ok - end, - Rows = qlc:next_answers(Cursor, PageSize), - qlc:delete_cursor(Cursor), - [{totalNum, TotalNum}, - {totalPage, total_page(TotalNum, PageSize)}, - {result, Rows}]. - -total_page(TotalNum, PageSize) -> - case TotalNum rem PageSize of - 0 -> TotalNum div PageSize; - _ -> (TotalNum div PageSize) + 1 - end. - -%%TODO: refactor later... -lookup_table(LookupFun, _PageNo, _PageSize) -> - Rows = LookupFun(), - Rows. - -%%-------------------------------------------------------------------- -%% mqtt -%%-------------------------------------------------------------------- -publish({ClientId, Topic, Payload, Qos, Retain}) -> - case validate(topic, Topic) of - true -> - Msg = emqttd_message:make(ClientId, Qos, Topic, Payload), - emqttd:publish(Msg#mqtt_message{retain = Retain}), - ok; - false -> - {error, format_error(Topic, "validate topic: ${0} fail")} - end. - -subscribe({ClientId, Topic, Qos}) -> - case validate(topic, Topic) of - true -> - case emqttd_sm:lookup_session(ClientId) of - undefined -> - {error, format_error(ClientId, "Clientid: ${0} not found")}; - #mqtt_session{sess_pid = SessPid} -> - emqttd_session:subscribe(SessPid, [{Topic, [{qos, Qos}]}]), - ok - end; - false -> - {error, format_error(Topic, "validate topic: ${0} fail")} - end. - -unsubscribe({ClientId, Topic}) -> - case validate(topic, Topic) of - true -> - case emqttd_sm:lookup_session(ClientId) of - undefined -> - {error, format_error(ClientId, "Clientid: ${0} not found")}; - #mqtt_session{sess_pid = SessPid} -> - emqttd_session:unsubscribe(SessPid, [{Topic, []}]), - ok - end; - false -> - {error, format_error(Topic, "validate topic: ${0} fail")} - end. - -%%-------------------------------------------------------------------- -%% manager API -%%-------------------------------------------------------------------- -kick_client(ClientId) -> - Result = [kick_client(Node, ClientId) || Node <- ekka_mnesia:running_nodes()], - lists:any(fun(Item) -> Item =:= ok end, Result). - -kick_client(Node, ClientId) when Node =:= node() -> - case emqttd_cm:lookup(ClientId) of - undefined -> error; - #mqtt_client{client_pid = Pid}-> emqttd_client:kick(Pid) - end; -kick_client(Node, ClientId) -> - rpc_call(Node, kick_client, [Node, ClientId]). - - -clean_acl_cache(ClientId, Topic) -> - Result = [clean_acl_cache(Node, ClientId, Topic) || Node <- ekka_mnesia:running_nodes()], - lists:any(fun(Item) -> Item =:= ok end, Result). - -clean_acl_cache(Node, ClientId, Topic) when Node =:= node() -> - case emqttd_cm:lookup(ClientId) of - undefined -> error; - #mqtt_client{client_pid = Pid}-> emqttd_client:clean_acl_cache(Pid, Topic) - end; -clean_acl_cache(Node, ClientId, Topic) -> - rpc_call(Node, clean_acl_cache, [Node, ClientId, Topic]). - -%%-------------------------------------------------------------------- -%% Config ENV -%%-------------------------------------------------------------------- -modify_config(App, Terms) -> - emqttd_config:write(App, Terms). - -modify_config(App, Key, Value) -> - Result = [modify_config(Node, App, Key, Value) || Node <- ekka_mnesia:running_nodes()], - lists:any(fun(Item) -> Item =:= ok end, Result). - -modify_config(Node, App, Key, Value) when Node =:= node() -> - emqttd_config:set(App, Key, Value); -modify_config(Node, App, Key, Value) -> - rpc_call(Node, modify_config, [Node, App, Key, Value]). - -get_configs() -> - [{Node, get_config(Node)} || Node <- ekka_mnesia:running_nodes()]. - -get_config(Node) when Node =:= node()-> - emqttd_cli_config:all_cfgs(); -get_config(Node) -> - rpc_call(Node, get_config, [Node]). - -get_plugin_config(PluginName) -> - emqttd_config:read(PluginName). -get_plugin_config(Node, PluginName) -> - rpc_call(Node, get_plugin_config, [PluginName]). - -modify_plugin_config(PluginName, Terms) -> - emqttd_config:write(PluginName, Terms). -modify_plugin_config(Node, PluginName, Terms) -> - rpc_call(Node, modify_plugin_config, [PluginName, Terms]). - -%%-------------------------------------------------------------------- -%% manager user API -%%-------------------------------------------------------------------- -check_user(undefined, _) -> - {error, "Username undefined"}; -check_user(_, undefined) -> - {error, "Password undefined"}; -check_user(Username, Password) -> - case mnesia:dirty_read(mqtt_admin, Username) of - [#mqtt_admin{password = <>}] -> - case Hash =:= md5_hash(Salt, Password) of - true -> ok; - false -> {error, "Password error"} - end; - [] -> - {error, "User not found"} - end. - -add_user(Username, Password, Tag) -> - Admin = #mqtt_admin{username = Username, - password = hash(Password), - tags = Tag}, - return(mnesia:transaction(fun add_user_/1, [Admin])). - -add_user_(Admin = #mqtt_admin{username = Username}) -> - case mnesia:wread({mqtt_admin, Username}) of - [] -> mnesia:write(Admin); - [_] -> {error, [{code, ?ERROR13}, {message, <<"User already exist">>}]} - end. - -user_list() -> - [row(Admin) || Admin <- ets:tab2list(mqtt_admin)]. - -lookup_user(Username) -> - Admin = mnesia:dirty_read(mqtt_admin, Username), - row(Admin). - -update_user(Username, Params) -> - case mnesia:dirty_read({mqtt_admin, Username}) of - [] -> - {error, [{code, ?ERROR5}, {message, <<"User not found">>}]}; - [User] -> - Admin = case proplists:get_value(<<"tags">>, Params) of - undefined -> User; - Tag -> User#mqtt_admin{tags = Tag} - end, - return(mnesia:transaction(fun() -> mnesia:write(Admin) end)) - end. - -remove_user(Username) -> - Trans = fun() -> - case lookup_user(Username) of - [] -> {error, [{code, ?ERROR5}, {message, <<"User not found">>}]}; - _ -> mnesia:delete({mqtt_admin, Username}) - end - end, - return(mnesia:transaction(Trans)). - -change_password(Username, OldPwd, NewPwd) -> - Trans = fun() -> - case mnesia:wread({mqtt_admin, Username}) of - [Admin = #mqtt_admin{password = <>}] -> - case Hash =:= md5_hash(Salt, OldPwd) of - true -> - mnesia:write(Admin#mqtt_admin{password = hash(NewPwd)}); - false -> - {error, [{code, ?ERROR14}, {message, <<"OldPassword error">>}]} - end; - [] -> - {error, [{code, ?ERROR5}, {message, <<"User not found">>}]} - end - end, - return(mnesia:transaction(Trans)). - -return({atomic, ok}) -> - ok; -return({atomic, Error}) -> - Error; -return({aborted, Reason}) -> - lager:error("Mnesia Transaction error:~p~n", [Reason]), - error. - -row(#mqtt_admin{username = Username, tags = Tags}) -> - [{username, Username}, {tags, Tags}]; -row([#mqtt_admin{username = Username, tags = Tags}]) -> - [{username, Username}, {tags, Tags}]; -row([]) ->[]. -%%-------------------------------------------------------------------- -%% Internel Functions. -%%-------------------------------------------------------------------- - -rpc_call(Node, Fun, Args) -> - case rpc:call(Node, ?MODULE, Fun, Args) of - {badrpc, Reason} -> {error, Reason}; - Res -> Res - end. - -kmg(Byte) when Byte > ?GB -> - float(Byte / ?GB, "G"); -kmg(Byte) when Byte > ?MB -> - float(Byte / ?MB, "M"); -kmg(Byte) when Byte > ?KB -> - float(Byte / ?MB, "K"); -kmg(Byte) -> - Byte. -float(F, S) -> - iolist_to_binary(io_lib:format("~.2f~s", [F, S])). - -validate(qos, Qos) -> - (Qos >= ?QOS_0) and (Qos =< ?QOS_2); - -validate(topic, Topic) -> - emqttd_topic:validate({name, Topic}). - -client_list(ClientId, PageNo, PageSize) when ?EMPTY_KEY(ClientId) -> - TotalNum = ets:info(mqtt_client, size), - Qh = qlc:q([R || R <- ets:table(mqtt_client)]), - query_table(Qh, PageNo, PageSize, TotalNum); - -client_list(ClientId, PageNo, PageSize) -> - Fun = fun() -> ets:lookup(mqtt_client, ClientId) end, - lookup_table(Fun, PageNo, PageSize). - -session_list(ClientId, PageNo, PageSize) when ?EMPTY_KEY(ClientId) -> - TotalNum = lists:sum([ets:info(Tab, size) || Tab <- [mqtt_local_session]]), - Qh = qlc:append([qlc:q([E || E <- ets:table(Tab)]) || Tab <- [mqtt_local_session]]), - query_table(Qh, PageNo, PageSize, TotalNum); - -session_list(ClientId, PageNo, PageSize) -> - MP = {ClientId, '_', '_', '_'}, - Fun = fun() -> lists:append([ets:match_object(Tab, MP) || Tab <- [mqtt_local_session]]) end, - lookup_table(Fun, PageNo, PageSize). - -subscription_list(Key, PageNo, PageSize) when ?EMPTY_KEY(Key) -> - TotalNum = ets:info(mqtt_subproperty, size), - Qh = qlc:q([E || E <- ets:table(mqtt_subproperty)]), - query_table(Qh, PageNo, PageSize, TotalNum); - -subscription_list(Key, PageNo, PageSize) -> - Fun = fun() -> ets:match_object(mqtt_subproperty, {{'_', {Key, '_'}}, '_'}) end, - lookup_table(Fun, PageNo, PageSize). - -route_list(Topic, PageNo, PageSize) when ?EMPTY_KEY(Topic) -> - Tables = [mqtt_route], - TotalNum = lists:sum([ets:info(Tab, size) || Tab <- [mqtt_route, mqtt_local_route]]), - Qh = qlc:append([qlc:q([E || E <- ets:table(Tab)]) || Tab <- Tables]), - Data = query_table(Qh, PageNo, PageSize, TotalNum), - Route = get_value(result, Data), - LocalRoute = local_route_list(Topic, PageNo, PageSize), - lists:keyreplace(result, 1, Data, {result, lists:append(Route, LocalRoute)}); - -route_list(Topic, PageNo, PageSize) -> - Tables = [mqtt_route], - Fun = fun() -> lists:append([ets:lookup(Tab, Topic) || Tab <- Tables]) end, - Route = lookup_table(Fun, PageNo, PageSize), - LocalRoute = local_route_list(Topic, PageNo, PageSize), - lists:append(Route, LocalRoute). - -local_route_list(Topic, PageNo, PageSize) when ?EMPTY_KEY(Topic) -> - TotalNum = lists:sum([ets:info(Tab, size) || Tab <- [mqtt_local_route]]), - Qh = qlc:append([qlc:q([E || E <- ets:table(Tab)]) || Tab <- [mqtt_local_route]]), - Data = query_table(Qh, PageNo, PageSize, TotalNum), - lists:map(fun({Topic1, Node}) -> {<<"$local/", Topic1/binary>>, Node} end, get_value(result, Data)); - -local_route_list(Topic, PageNo, PageSize) -> - Fun = fun() -> lists:append([ets:lookup(Tab, Topic) || Tab <- [mqtt_local_route]]) end, - Data = lookup_table(Fun, PageNo, PageSize), - lists:map(fun({Topic1, Node}) -> {<<"$local/", Topic1/binary>>, Node} end, Data). - - -format_error(Val, Msg) -> - re:replace(Msg, <<"\\$\\{[^}]+\\}">>, Val, [global, {return, binary}]). - -hash(Password) -> - SaltBin = salt(), - <>. - -md5_hash(SaltBin, Password) -> - erlang:md5(<>). - -salt() -> - seed(), - Salt = rand:uniform(16#ffffffff), - <>. - -seed() -> - rand:seed(exsplus, erlang:timestamp()). diff --git a/src/emqttd_misc.erl b/src/emqttd_misc.erl deleted file mode 100644 index 2224879ca..000000000 --- a/src/emqttd_misc.erl +++ /dev/null @@ -1,65 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_misc). - --author("Feng Lee "). - --export([merge_opts/2, start_timer/2, start_timer/3, cancel_timer/1, - proc_stats/0, proc_stats/1]). - -%% @doc Merge Options -merge_opts(Defaults, Options) -> - lists:foldl( - fun({Opt, Val}, Acc) -> - case lists:keymember(Opt, 1, Acc) of - true -> lists:keyreplace(Opt, 1, Acc, {Opt, Val}); - false -> [{Opt, Val}|Acc] - end; - (Opt, Acc) -> - case lists:member(Opt, Acc) of - true -> Acc; - false -> [Opt | Acc] - end - end, Defaults, Options). - --spec(start_timer(integer(), term()) -> reference()). -start_timer(Interval, Msg) -> - start_timer(Interval, self(), Msg). - --spec(start_timer(integer(), pid() | atom(), term()) -> reference()). -start_timer(Interval, Dest, Msg) -> - erlang:start_timer(Interval, Dest, Msg). - --spec(cancel_timer(undefined | reference()) -> ok). -cancel_timer(undefined) -> - ok; -cancel_timer(Timer) -> - case catch erlang:cancel_timer(Timer) of - false -> receive {timeout, Timer, _} -> ok after 0 -> ok end; - _ -> ok - end. - --spec(proc_stats() -> list()). -proc_stats() -> - proc_stats(self()). - --spec(proc_stats(pid()) -> list()). -proc_stats(Pid) -> - Stats = process_info(Pid, [message_queue_len, heap_size, reductions]), - {value, {_, V}, Stats1} = lists:keytake(message_queue_len, 1, Stats), - [{mailbox_len, V} | Stats1]. - diff --git a/src/emqttd_mqueue.erl b/src/emqttd_mqueue.erl deleted file mode 100644 index 92fda72f1..000000000 --- a/src/emqttd_mqueue.erl +++ /dev/null @@ -1,227 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc A Simple in-memory message queue. -%% -%% Notice that MQTT is not an enterprise messaging queue. MQTT assume that client -%% should be online in most of the time. -%% -%% This module implements a simple in-memory queue for MQTT persistent session. -%% -%% If the broker restarted or crashed, all the messages queued will be gone. -%% -%% Concept of Message Queue and Inflight Window: -%% -%% |<----------------- Max Len ----------------->| -%% ----------------------------------------------- -%% IN -> | Messages Queue | Inflight Window | -> Out -%% ----------------------------------------------- -%% |<--- Win Size --->| -%% -%% -%% 1. Inflight Window to store the messages delivered and awaiting for puback. -%% -%% 2. Enqueue messages when the inflight window is full. -%% -%% 3. If the queue is full, dropped qos0 messages if store_qos0 is true, -%% otherwise dropped the oldest one. -%% -%% @end - --module(emqttd_mqueue). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --import(proplists, [get_value/3]). - --export([new/3, type/1, name/1, is_empty/1, len/1, max_len/1, in/2, out/1, - dropped/1, stats/1]). - --define(LOW_WM, 0.2). - --define(HIGH_WM, 0.6). - --define(PQUEUE, priority_queue). - --type(priority() :: {iolist(), pos_integer()}). - --type(option() :: {type, simple | priority} - | {max_length, non_neg_integer()} %% Max queue length - | {priority, list(priority())} - | {low_watermark, float()} %% Low watermark - | {high_watermark, float()} %% High watermark - | {store_qos0, boolean()}). %% Queue Qos0? - --type(stat() :: {max_len, non_neg_integer()} - | {len, non_neg_integer()} - | {dropped, non_neg_integer()}). - --record(mqueue, {type :: simple | priority, - name, q :: queue:queue() | ?PQUEUE:q(), - %% priority table - pseq = 0, priorities = [], - %% len of simple queue - len = 0, max_len = 0, - low_wm = ?LOW_WM, high_wm = ?HIGH_WM, - qos0 = false, dropped = 0, - alarm_fun}). - --type(mqueue() :: #mqueue{}). - --export_type([mqueue/0, priority/0, option/0]). - -%% @doc New Queue. --spec(new(iolist(), list(option()), fun()) -> mqueue()). -new(Name, Opts, AlarmFun) -> - Type = get_value(type, Opts, simple), - MaxLen = get_value(max_length, Opts, 0), - init_q(#mqueue{type = Type, name = iolist_to_binary(Name), - len = 0, max_len = MaxLen, - low_wm = low_wm(MaxLen, Opts), - high_wm = high_wm(MaxLen, Opts), - qos0 = get_value(store_qos0, Opts, false), - alarm_fun = AlarmFun}, Opts). - -init_q(MQ = #mqueue{type = simple}, _Opts) -> - MQ#mqueue{q = queue:new()}; -init_q(MQ = #mqueue{type = priority}, Opts) -> - Priorities = get_value(priority, Opts, []), - init_p(Priorities, MQ#mqueue{q = ?PQUEUE:new()}). - -init_p([], MQ) -> - MQ; -init_p([{Topic, P} | L], MQ) -> - {_, MQ1} = insert_p(iolist_to_binary(Topic), P, MQ), - init_p(L, MQ1). - -insert_p(Topic, P, MQ = #mqueue{priorities = Tab, pseq = Seq}) -> - <> = <>, - {PInt, MQ#mqueue{priorities = [{Topic, PInt} | Tab], pseq = Seq + 1}}. - -low_wm(0, _Opts) -> - undefined; -low_wm(MaxLen, Opts) -> - round(MaxLen * get_value(low_watermark, Opts, ?LOW_WM)). - -high_wm(0, _Opts) -> - undefined; -high_wm(MaxLen, Opts) -> - round(MaxLen * get_value(high_watermark, Opts, ?HIGH_WM)). - --spec(name(mqueue()) -> iolist()). -name(#mqueue{name = Name}) -> - Name. - --spec(type(mqueue()) -> atom()). -type(#mqueue{type = Type}) -> - Type. - -is_empty(#mqueue{type = simple, len = Len}) -> Len =:= 0; -is_empty(#mqueue{type = priority, q = Q}) -> ?PQUEUE:is_empty(Q). - -len(#mqueue{type = simple, len = Len}) -> Len; -len(#mqueue{type = priority, q = Q}) -> ?PQUEUE:len(Q). - -max_len(#mqueue{max_len = MaxLen}) -> MaxLen. - -%% @doc Dropped of the mqueue --spec(dropped(mqueue()) -> non_neg_integer()). -dropped(#mqueue{dropped = Dropped}) -> Dropped. - -%% @doc Stats of the mqueue --spec(stats(mqueue()) -> [stat()]). -stats(#mqueue{type = Type, q = Q, max_len = MaxLen, len = Len, dropped = Dropped}) -> - [{len, case Type of - simple -> Len; - priority -> ?PQUEUE:len(Q) - end} | [{max_len, MaxLen}, {dropped, Dropped}]]. - -%% @doc Enqueue a message. --spec(in(mqtt_message(), mqueue()) -> mqueue()). -in(#mqtt_message{qos = ?QOS_0}, MQ = #mqueue{qos0 = false}) -> - MQ; -in(Msg, MQ = #mqueue{type = simple, q = Q, len = Len, max_len = 0}) -> - MQ#mqueue{q = queue:in(Msg, Q), len = Len + 1}; -in(Msg, MQ = #mqueue{type = simple, q = Q, len = Len, max_len = MaxLen, dropped = Dropped}) - when Len >= MaxLen -> - {{value, _Old}, Q2} = queue:out(Q), - MQ#mqueue{q = queue:in(Msg, Q2), dropped = Dropped +1}; -in(Msg, MQ = #mqueue{type = simple, q = Q, len = Len}) -> - maybe_set_alarm(MQ#mqueue{q = queue:in(Msg, Q), len = Len + 1}); - -in(Msg = #mqtt_message{topic = Topic}, MQ = #mqueue{type = priority, q = Q, - priorities = Priorities, - max_len = 0}) -> - case lists:keysearch(Topic, 1, Priorities) of - {value, {_, Pri}} -> - MQ#mqueue{q = ?PQUEUE:in(Msg, Pri, Q)}; - false -> - {Pri, MQ1} = insert_p(Topic, 0, MQ), - MQ1#mqueue{q = ?PQUEUE:in(Msg, Pri, Q)} - end; -in(Msg = #mqtt_message{topic = Topic}, MQ = #mqueue{type = priority, q = Q, - priorities = Priorities, - max_len = MaxLen}) -> - case lists:keysearch(Topic, 1, Priorities) of - {value, {_, Pri}} -> - case ?PQUEUE:plen(Pri, Q) >= MaxLen of - true -> - {_, Q1} = ?PQUEUE:out(Pri, Q), - MQ#mqueue{q = ?PQUEUE:in(Msg, Pri, Q1)}; - false -> - MQ#mqueue{q = ?PQUEUE:in(Msg, Pri, Q)} - end; - false -> - {Pri, MQ1} = insert_p(Topic, 0, MQ), - MQ1#mqueue{q = ?PQUEUE:in(Msg, Pri, Q)} - end. - -out(MQ = #mqueue{type = simple, len = 0}) -> - {empty, MQ}; -out(MQ = #mqueue{type = simple, q = Q, len = Len, max_len = 0}) -> - {R, Q2} = queue:out(Q), - {R, MQ#mqueue{q = Q2, len = Len - 1}}; -out(MQ = #mqueue{type = simple, q = Q, len = Len}) -> - {R, Q2} = queue:out(Q), - {R, maybe_clear_alarm(MQ#mqueue{q = Q2, len = Len - 1})}; -out(MQ = #mqueue{type = priority, q = Q}) -> - {R, Q2} = ?PQUEUE:out(Q), - {R, MQ#mqueue{q = Q2}}. - -maybe_set_alarm(MQ = #mqueue{high_wm = undefined}) -> - MQ; -maybe_set_alarm(MQ = #mqueue{name = Name, len = Len, high_wm = HighWM, alarm_fun = AlarmFun}) - when Len > HighWM -> - Alarm = #mqtt_alarm{id = iolist_to_binary(["queue_high_watermark.", Name]), - severity = warning, - title = io_lib:format("Queue ~s high-water mark", [Name]), - summary = io_lib:format("queue len ~p > high_watermark ~p", [Len, HighWM])}, - MQ#mqueue{alarm_fun = AlarmFun(alert, Alarm)}; -maybe_set_alarm(MQ) -> - MQ. - -maybe_clear_alarm(MQ = #mqueue{low_wm = undefined}) -> - MQ; -maybe_clear_alarm(MQ = #mqueue{name = Name, len = Len, low_wm = LowWM, alarm_fun = AlarmFun}) - when Len < LowWM -> - MQ#mqueue{alarm_fun = AlarmFun(clear, list_to_binary(["queue_high_watermark.", Name]))}; -maybe_clear_alarm(MQ) -> - MQ. - diff --git a/src/emqttd_net.erl b/src/emqttd_net.erl deleted file mode 100644 index 9da9cd287..000000000 --- a/src/emqttd_net.erl +++ /dev/null @@ -1,232 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_net). - --include_lib("kernel/include/inet.hrl"). - --export([tcp_name/3, tcp_host/1, getopts/2, setopts/2, getaddr/2, - port_to_listeners/1]). - --export([peername/1, sockname/1, format/2, format/1, ntoa/1, - connection_string/2]). - --define(FIRST_TEST_BIND_PORT, 10000). - -%%-------------------------------------------------------------------- - -%% inet_parse:address takes care of ip string, like "0.0.0.0" -%% inet:getaddr returns immediately for ip tuple {0,0,0,0}, -%% and runs 'inet_gethost' port process for dns lookups. -%% On Windows inet:getaddr runs dns resolver for ip string, which may fail. -getaddr(Host, Family) -> - case inet_parse:address(Host) of - {ok, IPAddress} -> [{IPAddress, resolve_family(IPAddress, Family)}]; - {error, _} -> gethostaddr(Host, Family) - end. - -gethostaddr(Host, auto) -> - Lookups = [{Family, inet:getaddr(Host, Family)} || Family <- [inet, inet6]], - case [{IP, Family} || {Family, {ok, IP}} <- Lookups] of - [] -> host_lookup_error(Host, Lookups); - IPs -> IPs - end; - -gethostaddr(Host, Family) -> - case inet:getaddr(Host, Family) of - {ok, IPAddress} -> [{IPAddress, Family}]; - {error, Reason} -> host_lookup_error(Host, Reason) - end. - -host_lookup_error(Host, Reason) -> - error_logger:error_msg("invalid host ~p - ~p~n", [Host, Reason]), - throw({error, {invalid_host, Host, Reason}}). - -resolve_family({_,_,_,_}, auto) -> inet; -resolve_family({_,_,_,_,_,_,_,_}, auto) -> inet6; -resolve_family(IP, auto) -> throw({error, {strange_family, IP}}); -resolve_family(_, F) -> F. - -%%-------------------------------------------------------------------- - -%% There are three kinds of machine (for our purposes). -%% -%% * Those which treat IPv4 addresses as a special kind of IPv6 address -%% ("Single stack") -%% - Linux by default, Windows Vista and later -%% - We also treat any (hypothetical?) IPv6-only machine the same way -%% * Those which consider IPv6 and IPv4 to be completely separate things -%% ("Dual stack") -%% - OpenBSD, Windows XP / 2003, Linux if so configured -%% * Those which do not support IPv6. -%% - Ancient/weird OSes, Linux if so configured -%% -%% How to reconfigure Linux to test this: -%% Single stack (default): -%% echo 0 > /proc/sys/net/ipv6/bindv6only -%% Dual stack: -%% echo 1 > /proc/sys/net/ipv6/bindv6only -%% IPv4 only: -%% add ipv6.disable=1 to GRUB_CMDLINE_LINUX_DEFAULT in /etc/default/grub then -%% sudo update-grub && sudo reboot -%% -%% This matters in (and only in) the case where the sysadmin (or the -%% app descriptor) has only supplied a port and we wish to bind to -%% "all addresses". This means different things depending on whether -%% we're single or dual stack. On single stack binding to "::" -%% implicitly includes all IPv4 addresses, and subsequently attempting -%% to bind to "0.0.0.0" will fail. On dual stack, binding to "::" will -%% only bind to IPv6 addresses, and we need another listener bound to -%% "0.0.0.0" for IPv4. Finally, on IPv4-only systems we of course only -%% want to bind to "0.0.0.0". -%% -%% Unfortunately it seems there is no way to detect single vs dual stack -%% apart from attempting to bind to the port. -port_to_listeners(Port) -> - IPv4 = {"0.0.0.0", Port, inet}, - IPv6 = {"::", Port, inet6}, - case ipv6_status(?FIRST_TEST_BIND_PORT) of - single_stack -> [IPv6]; - ipv6_only -> [IPv6]; - dual_stack -> [IPv6, IPv4]; - ipv4_only -> [IPv4] - end. - -ipv6_status(TestPort) -> - IPv4 = [inet, {ip, {0,0,0,0}}], - IPv6 = [inet6, {ip, {0,0,0,0,0,0,0,0}}], - case gen_tcp:listen(TestPort, IPv6) of - {ok, LSock6} -> - case gen_tcp:listen(TestPort, IPv4) of - {ok, LSock4} -> - %% Dual stack - gen_tcp:close(LSock6), - gen_tcp:close(LSock4), - dual_stack; - %% Checking the error here would only let us - %% distinguish single stack IPv6 / IPv4 vs IPv6 only, - %% which we figure out below anyway. - {error, _} -> - gen_tcp:close(LSock6), - case gen_tcp:listen(TestPort, IPv4) of - %% Single stack - {ok, LSock4} -> gen_tcp:close(LSock4), - single_stack; - %% IPv6-only machine. Welcome to the future. - {error, eafnosupport} -> ipv6_only; %% Linux - {error, eprotonosupport}-> ipv6_only; %% FreeBSD - %% Dual stack machine with something already - %% on IPv4. - {error, _} -> ipv6_status(TestPort + 1) - end - end; - %% IPv4-only machine. Welcome to the 90s. - {error, eafnosupport} -> %% Linux - ipv4_only; - {error, eprotonosupport} -> %% FreeBSD - ipv4_only; - %% Port in use - {error, _} -> - ipv6_status(TestPort + 1) - end. - -tcp_name(Prefix, IPAddress, Port) - when is_atom(Prefix) andalso is_number(Port) -> - list_to_atom( - lists:flatten( - io_lib:format( - "~w_~s:~w", [Prefix, inet_parse:ntoa(IPAddress), Port]))). - -connection_string(Sock, Direction) -> - case socket_ends(Sock, Direction) of - {ok, {FromAddress, FromPort, ToAddress, ToPort}} -> - {ok, lists:flatten( - io_lib:format( - "~s:~p -> ~s:~p", - [maybe_ntoab(FromAddress), FromPort, - maybe_ntoab(ToAddress), ToPort]))}; - Error -> - Error - end. - -socket_ends(Sock, Direction) -> - {From, To} = sock_funs(Direction), - case {From(Sock), To(Sock)} of - {{ok, {FromAddress, FromPort}}, {ok, {ToAddress, ToPort}}} -> - {ok, {rdns(FromAddress), FromPort, - rdns(ToAddress), ToPort}}; - {{error, _Reason} = Error, _} -> - Error; - {_, {error, _Reason} = Error} -> - Error - end. - -maybe_ntoab(Addr) when is_tuple(Addr) -> ntoab(Addr); -maybe_ntoab(Host) -> Host. - -rdns(Addr) -> Addr. - -sock_funs(inbound) -> {fun peername/1, fun sockname/1}; -sock_funs(outbound) -> {fun sockname/1, fun peername/1}. - -getopts(Sock, Options) when is_port(Sock) -> - inet:getopts(Sock, Options). - -setopts(Sock, Options) when is_port(Sock) -> - inet:setopts(Sock, Options). - -sockname(Sock) when is_port(Sock) -> inet:sockname(Sock). - -peername(Sock) when is_port(Sock) -> inet:peername(Sock). - -format(sockname, SockName) -> - format(SockName); -format(peername, PeerName) -> - format(PeerName). -format({Addr, Port}) -> - lists:flatten(io_lib:format("~s:~p", [maybe_ntoab(Addr), Port])). - -ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> - inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}); -ntoa(IP) -> - inet_parse:ntoa(IP). - -ntoab(IP) -> - Str = ntoa(IP), - case string:str(Str, ":") of - 0 -> Str; - _ -> "[" ++ Str ++ "]" - end. - -tcp_host({0,0,0,0}) -> - hostname(); - -tcp_host({0,0,0,0,0,0,0,0}) -> - hostname(); - -tcp_host(IPAddress) -> - case inet:gethostbyaddr(IPAddress) of - {ok, #hostent{h_name = Name}} -> Name; - {error, _Reason} -> ntoa(IPAddress) - end. - -hostname() -> - {ok, Hostname} = inet:gethostname(), - case inet:gethostbyname(Hostname) of - {ok, #hostent{h_name = Name}} -> Name; - {error, _Reason} -> Hostname - end. - diff --git a/src/emqttd_packet.erl b/src/emqttd_packet.erl deleted file mode 100644 index d909089d1..000000000 --- a/src/emqttd_packet.erl +++ /dev/null @@ -1,130 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_packet). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - -%% API --export([protocol_name/1, type_name/1, connack_name/1]). - --export([format/1]). - -%% @doc Protocol name of version --spec(protocol_name(mqtt_vsn()) -> binary()). -protocol_name(?MQTT_PROTO_V3) -> <<"MQIsdp">>; -protocol_name(?MQTT_PROTO_V4) -> <<"MQTT">>; -protocol_name(?MQTT_PROTO_V5) -> <<"MQTT">>. - -%% @doc Name of MQTT packet type --spec(type_name(mqtt_packet_type()) -> atom()). -type_name(Type) when Type > ?RESERVED andalso Type =< ?AUTH -> - lists:nth(Type, ?TYPE_NAMES). - -%% @doc Connack Name --spec(connack_name(mqtt_connack()) -> atom()). -connack_name(?CONNACK_ACCEPT) -> 'CONNACK_ACCEPT'; -connack_name(?CONNACK_PROTO_VER) -> 'CONNACK_PROTO_VER'; -connack_name(?CONNACK_INVALID_ID) -> 'CONNACK_INVALID_ID'; -connack_name(?CONNACK_SERVER) -> 'CONNACK_SERVER'; -connack_name(?CONNACK_CREDENTIALS) -> 'CONNACK_CREDENTIALS'; -connack_name(?CONNACK_AUTH) -> 'CONNACK_AUTH'. - -%% @doc Format packet --spec(format(mqtt_packet()) -> iolist()). -format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}) -> - format_header(Header, format_variable(Variable, Payload)). - -format_header(#mqtt_packet_header{type = Type, - dup = Dup, - qos = QoS, - retain = Retain}, S) -> - S1 = if - S == undefined -> <<>>; - true -> [", ", S] - end, - io_lib:format("~s(Q~p, R~p, D~p~s)", [type_name(Type), QoS, i(Retain), i(Dup), S1]). - -format_variable(undefined, _) -> - undefined; -format_variable(Variable, undefined) -> - format_variable(Variable); -format_variable(Variable, Payload) -> - io_lib:format("~s, Payload=~p", [format_variable(Variable), Payload]). - -format_variable(#mqtt_packet_connect{ - proto_ver = ProtoVer, - proto_name = ProtoName, - will_retain = WillRetain, - will_qos = WillQoS, - will_flag = WillFlag, - clean_sess = CleanSess, - keep_alive = KeepAlive, - client_id = ClientId, - will_topic = WillTopic, - will_msg = WillMsg, - username = Username, - password = Password}) -> - Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanSess=~s, KeepAlive=~p, Username=~s, Password=~s", - Args = [ClientId, ProtoName, ProtoVer, CleanSess, KeepAlive, Username, format_password(Password)], - {Format1, Args1} = if - WillFlag -> { Format ++ ", Will(Q~p, R~p, Topic=~s, Msg=~s)", - Args ++ [WillQoS, i(WillRetain), WillTopic, WillMsg] }; - true -> {Format, Args} - end, - io_lib:format(Format1, Args1); - -format_variable(#mqtt_packet_connack{ack_flags = AckFlags, - return_code = ReturnCode}) -> - io_lib:format("AckFlags=~p, ReturnCode=~p", [AckFlags, ReturnCode]); - -format_variable(#mqtt_packet_publish{topic_name = TopicName, - packet_id = PacketId}) -> - io_lib:format("Topic=~s, PacketId=~p", [TopicName, PacketId]); - -format_variable(#mqtt_packet_puback{packet_id = PacketId}) -> - io_lib:format("PacketId=~p", [PacketId]); - -format_variable(#mqtt_packet_subscribe{packet_id = PacketId, - topic_table = TopicTable}) -> - io_lib:format("PacketId=~p, TopicTable=~p", [PacketId, TopicTable]); - -format_variable(#mqtt_packet_unsubscribe{packet_id = PacketId, - topics = Topics}) -> - io_lib:format("PacketId=~p, Topics=~p", [PacketId, Topics]); - -format_variable(#mqtt_packet_suback{packet_id = PacketId, - qos_table = QosTable}) -> - io_lib:format("PacketId=~p, QosTable=~p", [PacketId, QosTable]); - -format_variable(#mqtt_packet_unsuback{packet_id = PacketId}) -> - io_lib:format("PacketId=~p", [PacketId]); - -format_variable(PacketId) when is_integer(PacketId) -> - io_lib:format("PacketId=~p", [PacketId]); - -format_variable(undefined) -> undefined. - -format_password(undefined) -> undefined; -format_password(_Password) -> '******'. - -i(true) -> 1; -i(false) -> 0; -i(I) when is_integer(I) -> I. diff --git a/src/emqttd_parser.erl b/src/emqttd_parser.erl deleted file mode 100644 index e9277a7c6..000000000 --- a/src/emqttd_parser.erl +++ /dev/null @@ -1,231 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc MQTT Packet Parser --module(emqttd_parser). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - -%% API --export([initial_state/0, initial_state/1, parse/2]). - --type(max_packet_size() :: 1..?MAX_PACKET_SIZE). - --spec(initial_state() -> {none, max_packet_size()}). -initial_state() -> - initial_state(?MAX_PACKET_SIZE). - -%% @doc Initialize a parser --spec(initial_state(max_packet_size()) -> {none, max_packet_size()}). -initial_state(MaxSize) -> - {none, MaxSize}. - -%% @doc Parse MQTT Packet --spec(parse(binary(), {none, pos_integer()} | fun()) - -> {ok, mqtt_packet()} | {error, term()} | {more, fun()}). -parse(<<>>, {none, MaxLen}) -> - {more, fun(Bin) -> parse(Bin, {none, MaxLen}) end}; -parse(<>, {none, Limit}) -> - parse_remaining_len(Rest, #mqtt_packet_header{type = Type, - dup = bool(Dup), - qos = fixqos(Type, QoS), - retain = bool(Retain)}, Limit); -parse(Bin, Cont) -> Cont(Bin). - -parse_remaining_len(<<>>, Header, Limit) -> - {more, fun(Bin) -> parse_remaining_len(Bin, Header, Limit) end}; -parse_remaining_len(Rest, Header, Limit) -> - parse_remaining_len(Rest, Header, 1, 0, Limit). - -parse_remaining_len(_Bin, _Header, _Multiplier, Length, MaxLen) - when Length > MaxLen -> - {error, invalid_mqtt_frame_len}; -parse_remaining_len(<<>>, Header, Multiplier, Length, Limit) -> - {more, fun(Bin) -> parse_remaining_len(Bin, Header, Multiplier, Length, Limit) end}; -%% optimize: match PUBACK, PUBREC, PUBREL, PUBCOMP, UNSUBACK... -parse_remaining_len(<<0:1, 2:7, Rest/binary>>, Header, 1, 0, _Limit) -> - parse_frame(Rest, Header, 2); -%% optimize: match PINGREQ... -parse_remaining_len(<<0:8, Rest/binary>>, Header, 1, 0, _Limit) -> - parse_frame(Rest, Header, 0); -parse_remaining_len(<<1:1, Len:7, Rest/binary>>, Header, Multiplier, Value, Limit) -> - parse_remaining_len(Rest, Header, Multiplier * ?HIGHBIT, Value + Len * Multiplier, Limit); -parse_remaining_len(<<0:1, Len:7, Rest/binary>>, Header, Multiplier, Value, MaxLen) -> - FrameLen = Value + Len * Multiplier, - if - FrameLen > MaxLen -> {error, invalid_mqtt_frame_len}; - true -> parse_frame(Rest, Header, FrameLen) - end. - -parse_frame(Bin, #mqtt_packet_header{type = Type, qos = Qos} = Header, Length) -> - case {Type, Bin} of - {?CONNECT, <>} -> - {ProtoName, Rest1} = parse_utf(FrameBin), - %% Fix mosquitto bridge: 0x83, 0x84 - <> = Rest1, - <> = Rest2, - {ClientId, Rest4} = parse_utf(Rest3), - {WillTopic, Rest5} = parse_utf(Rest4, WillFlag), - {WillMsg, Rest6} = parse_msg(Rest5, WillFlag), - {UserName, Rest7} = parse_utf(Rest6, UsernameFlag), - {PasssWord, <<>>} = parse_utf(Rest7, PasswordFlag), - case protocol_name_approved(ProtoVersion, ProtoName) of - true -> - wrap(Header, - #mqtt_packet_connect{ - proto_ver = ProtoVersion, - proto_name = ProtoName, - will_retain = bool(WillRetain), - will_qos = WillQos, - will_flag = bool(WillFlag), - clean_sess = bool(CleanSess), - keep_alive = KeepAlive, - client_id = ClientId, - will_topic = WillTopic, - will_msg = WillMsg, - username = UserName, - password = PasssWord, - is_bridge = (BridgeTag =:= 8)}, Rest); - false -> - {error, protocol_header_corrupt} - end; - %{?CONNACK, <>} -> - % <<_Reserved:7, SP:1, ReturnCode:8>> = FrameBin, - % wrap(Header, #mqtt_packet_connack{ack_flags = SP, - % return_code = ReturnCode }, Rest); - {?PUBLISH, <>} -> - {TopicName, Rest1} = parse_utf(FrameBin), - {PacketId, Payload} = case Qos of - 0 -> {undefined, Rest1}; - _ -> <> = Rest1, - {Id, R} - end, - wrap(fixdup(Header), #mqtt_packet_publish{topic_name = TopicName, - packet_id = PacketId}, - Payload, Rest); - {?PUBACK, <>} -> - <> = FrameBin, - wrap(Header, #mqtt_packet_puback{packet_id = PacketId}, Rest); - {?PUBREC, <>} -> - <> = FrameBin, - wrap(Header, #mqtt_packet_puback{packet_id = PacketId}, Rest); - {?PUBREL, <>} -> - %% 1 = Qos, - <> = FrameBin, - wrap(Header, #mqtt_packet_puback{packet_id = PacketId}, Rest); - {?PUBCOMP, <>} -> - <> = FrameBin, - wrap(Header, #mqtt_packet_puback{packet_id = PacketId}, Rest); - {?SUBSCRIBE, <>} -> - %% 1 = Qos, - <> = FrameBin, - TopicTable = parse_topics(?SUBSCRIBE, Rest1, []), - wrap(Header, #mqtt_packet_subscribe{packet_id = PacketId, - topic_table = TopicTable}, Rest); - %{?SUBACK, <>} -> - % <> = FrameBin, - % wrap(Header, #mqtt_packet_suback{packet_id = PacketId, - % qos_table = parse_qos(Rest1, []) }, Rest); - {?UNSUBSCRIBE, <>} -> - %% 1 = Qos, - <> = FrameBin, - Topics = parse_topics(?UNSUBSCRIBE, Rest1, []), - wrap(Header, #mqtt_packet_unsubscribe{packet_id = PacketId, - topics = Topics}, Rest); - %{?UNSUBACK, <>} -> - % <> = FrameBin, - % wrap(Header, #mqtt_packet_unsuback { packet_id = PacketId }, Rest); - {?PINGREQ, Rest} -> - Length = 0, - wrap(Header, Rest); - %{?PINGRESP, Rest} -> - % Length = 0, - % wrap(Header, Rest); - {?DISCONNECT, Rest} -> - Length = 0, - wrap(Header, Rest); - {_, TooShortBin} -> - {more, fun(BinMore) -> - parse_frame(<>, - Header, Length) - end} - end. - -wrap(Header, Variable, Payload, Rest) -> - {ok, #mqtt_packet{header = Header, variable = Variable, payload = Payload}, Rest}. -wrap(Header, Variable, Rest) -> - {ok, #mqtt_packet{header = Header, variable = Variable}, Rest}. -wrap(Header, Rest) -> - {ok, #mqtt_packet{header = Header}, Rest}. - -%client function -%parse_qos(<<>>, Acc) -> -% lists:reverse(Acc); -%parse_qos(<>, Acc) -> -% parse_qos(Rest, [QoS | Acc]). - -parse_topics(_, <<>>, Topics) -> - lists:reverse(Topics); -parse_topics(?SUBSCRIBE = Sub, Bin, Topics) -> - {Name, <<_:6, QoS:2, Rest/binary>>} = parse_utf(Bin), - parse_topics(Sub, Rest, [{Name, QoS}| Topics]); -parse_topics(?UNSUBSCRIBE = Sub, Bin, Topics) -> - {Name, <>} = parse_utf(Bin), - parse_topics(Sub, Rest, [Name | Topics]). - -parse_utf(Bin, 0) -> - {undefined, Bin}; -parse_utf(Bin, _) -> - parse_utf(Bin). - -parse_utf(<>) -> - {Str, Rest}. - -parse_msg(Bin, 0) -> - {undefined, Bin}; -parse_msg(<>, _) -> - {Msg, Rest}. - -bool(0) -> false; -bool(1) -> true. - -protocol_name_approved(Ver, Name) -> - lists:member({Ver, Name}, ?PROTOCOL_NAMES). - -%% Fix Issue#575 -fixqos(?PUBREL, 0) -> 1; -fixqos(?SUBSCRIBE, 0) -> 1; -fixqos(?UNSUBSCRIBE, 0) -> 1; -fixqos(_Type, QoS) -> QoS. - -%% Fix Issue#1319 -fixdup(Header = #mqtt_packet_header{qos = ?QOS0, dup = true}) -> - Header#mqtt_packet_header{dup = false}; -fixdup(Header = #mqtt_packet_header{qos = ?QOS2, dup = true}) -> - Header#mqtt_packet_header{dup = false}; -fixdup(Header) -> Header. diff --git a/src/emqttd_pmon.erl b/src/emqttd_pmon.erl deleted file mode 100644 index 00cb9a4c3..000000000 --- a/src/emqttd_pmon.erl +++ /dev/null @@ -1,53 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_pmon). - --author("Feng Lee "). - --export([new/0, monitor/2, demonitor/2, erase/2]). - --type(pmon() :: {?MODULE, map()}). - --export_type([pmon/0]). - -new() -> - {?MODULE, [maps:new()]}. - --spec(monitor(pid(), pmon()) -> pmon()). -monitor(Pid, PM = {?MODULE, [M]}) -> - case maps:is_key(Pid, M) of - true -> - PM; - false -> - Ref = erlang:monitor(process, Pid), - {?MODULE, [maps:put(Pid, Ref, M)]} - end. - --spec(demonitor(pid(), pmon()) -> pmon()). -demonitor(Pid, PM = {?MODULE, [M]}) -> - case maps:find(Pid, M) of - {ok, Ref} -> - erlang:demonitor(Ref, [flush]), - {?MODULE, [maps:remove(Pid, M)]}; - error -> - PM - end. - --spec(erase(pid(), pmon()) -> pmon()). -erase(Pid, {?MODULE, [M]}) -> - {?MODULE, [maps:remove(Pid, M)]}. - diff --git a/src/emqttd_pooler.erl b/src/emqttd_pooler.erl deleted file mode 100644 index fdde12a66..000000000 --- a/src/emqttd_pooler.erl +++ /dev/null @@ -1,98 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_pooler). - --behaviour(gen_server). - --include("emqttd_internal.hrl"). - -%% Start the pool supervisor --export([start_link/0]). - -%% API Exports --export([start_link/2, submit/1, async_submit/1]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {pool, id}). - -%% @doc Start Pooler Supervisor. -start_link() -> - emqttd_pool_sup:start_link(pooler, random, {?MODULE, start_link, []}). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - --spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}). -start_link(Pool, Id) -> - gen_server:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id], []). - -%% @doc Submit work to pooler -submit(Fun) -> gen_server:call(worker(), {submit, Fun}, infinity). - -%% @doc Submit work to pooler asynchronously -async_submit(Fun) -> - gen_server:cast(worker(), {async_submit, Fun}). - -worker() -> - gproc_pool:pick_worker(pooler). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id]) -> - ?GPROC_POOL(join, Pool, Id), - {ok, #state{pool = Pool, id = Id}}. - -handle_call({submit, Fun}, _From, State) -> - {reply, run(Fun), State}; - -handle_call(_Req, _From, State) -> - {reply, ok, State}. - -handle_cast({async_submit, Fun}, State) -> - try run(Fun) - catch _:Error -> - lager:error("Pooler Error: ~p, ~p", [Error, erlang:get_stacktrace()]) - end, - {noreply, State}; - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - ?GPROC_POOL(leave, Pool, Id), ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -run({M, F, A}) -> - erlang:apply(M, F, A); -run(Fun) when is_function(Fun) -> - Fun(). - diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl deleted file mode 100644 index c2399f6a3..000000000 --- a/src/emqttd_protocol.erl +++ /dev/null @@ -1,598 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_protocol). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --include("emqttd_internal.hrl"). - --import(proplists, [get_value/2, get_value/3]). - -%% API --export([init/3, init/4, info/1, stats/1, clientid/1, client/1, session/1]). - --export([subscribe/2, unsubscribe/2, pubrel/2, shutdown/2]). - --export([received/2, send/2]). - --export([process/2]). - --record(proto_stats, {enable_stats = false, recv_pkt = 0, recv_msg = 0, - send_pkt = 0, send_msg = 0}). - -%% Protocol State -%% ws_initial_headers: Headers from first HTTP request for WebSocket Client. --record(proto_state, {peername, sendfun, connected = false, client_id, client_pid, - clean_sess, proto_ver, proto_name, username, is_superuser, - will_msg, keepalive, keepalive_backoff, max_clientid_len, - session, stats_data, mountpoint, ws_initial_headers, - peercert_username, is_bridge, connected_at}). - --type(proto_state() :: #proto_state{}). - --define(INFO_KEYS, [client_id, username, clean_sess, proto_ver, proto_name, - keepalive, will_msg, ws_initial_headers, mountpoint, - peercert_username, connected_at]). - --define(STATS_KEYS, [recv_pkt, recv_msg, send_pkt, send_msg]). - --define(LOG(Level, Format, Args, State), - lager:Level([{client, State#proto_state.client_id}], "Client(~s@~s): " ++ Format, - [State#proto_state.client_id, esockd_net:format(State#proto_state.peername) | Args])). - -%% @doc Init protocol -init(Peername, SendFun, Opts) -> - Backoff = get_value(keepalive_backoff, Opts, 1.25), - EnableStats = get_value(client_enable_stats, Opts, false), - MaxLen = get_value(max_clientid_len, Opts, ?MAX_CLIENTID_LEN), - WsInitialHeaders = get_value(ws_initial_headers, Opts), - #proto_state{peername = Peername, - sendfun = SendFun, - max_clientid_len = MaxLen, - is_superuser = false, - client_pid = self(), - peercert_username = undefined, - ws_initial_headers = WsInitialHeaders, - keepalive_backoff = Backoff, - stats_data = #proto_stats{enable_stats = EnableStats}}. - -init(Conn, Peername, SendFun, Opts) -> - enrich_opt(Conn:opts(), Conn, init(Peername, SendFun, Opts)). - -enrich_opt([], _Conn, State) -> - State; -enrich_opt([{mountpoint, MountPoint} | ConnOpts], Conn, State) -> - enrich_opt(ConnOpts, Conn, State#proto_state{mountpoint = MountPoint}); -enrich_opt([{peer_cert_as_username, N} | ConnOpts], Conn, State) -> - enrich_opt(ConnOpts, Conn, State#proto_state{peercert_username = peercert_username(N, Conn)}); -enrich_opt([_ | ConnOpts], Conn, State) -> - enrich_opt(ConnOpts, Conn, State). - -peercert_username(cn, Conn) -> - Conn:peer_cert_common_name(); -peercert_username(dn, Conn) -> - Conn:peer_cert_subject(). - -repl_username_with_peercert(State = #proto_state{peercert_username = undefined}) -> - State; -repl_username_with_peercert(State = #proto_state{peercert_username = PeerCert}) -> - State#proto_state{username = PeerCert}. - -info(ProtoState) -> - ?record_to_proplist(proto_state, ProtoState, ?INFO_KEYS). - -stats(#proto_state{stats_data = Stats}) -> - tl(?record_to_proplist(proto_stats, Stats)). - -clientid(#proto_state{client_id = ClientId}) -> - ClientId. - -client(#proto_state{client_id = ClientId, - client_pid = ClientPid, - peername = Peername, - username = Username, - clean_sess = CleanSess, - proto_ver = ProtoVer, - keepalive = Keepalive, - will_msg = WillMsg, - ws_initial_headers = WsInitialHeaders, - mountpoint = MountPoint, - connected_at = Time}) -> - WillTopic = if - WillMsg =:= undefined -> undefined; - true -> WillMsg#mqtt_message.topic - end, - #mqtt_client{client_id = ClientId, - client_pid = ClientPid, - username = Username, - peername = Peername, - clean_sess = CleanSess, - proto_ver = ProtoVer, - keepalive = Keepalive, - will_topic = WillTopic, - ws_initial_headers = WsInitialHeaders, - mountpoint = MountPoint, - connected_at = Time}. - -session(#proto_state{session = Session}) -> - Session. - -%% CONNECT – Client requests a connection to a Server - -%% A Client can only send the CONNECT Packet once over a Network Connection. --spec(received(mqtt_packet(), proto_state()) -> {ok, proto_state()} | {error, term()}). -received(Packet = ?PACKET(?CONNECT), - State = #proto_state{connected = false, stats_data = Stats}) -> - trace(recv, Packet, State), Stats1 = inc_stats(recv, ?CONNECT, Stats), - process(Packet, State#proto_state{connected = true, stats_data = Stats1}); - -received(?PACKET(?CONNECT), State = #proto_state{connected = true}) -> - {error, protocol_bad_connect, State}; - -%% Received other packets when CONNECT not arrived. -received(_Packet, State = #proto_state{connected = false}) -> - {error, protocol_not_connected, State}; - -received(Packet = ?PACKET(Type), State = #proto_state{stats_data = Stats}) -> - trace(recv, Packet, State), Stats1 = inc_stats(recv, Type, Stats), - case validate_packet(Packet) of - ok -> - process(Packet, State#proto_state{stats_data = Stats1}); - {error, Reason} -> - {error, Reason, State} - end. - -subscribe(RawTopicTable, ProtoState = #proto_state{client_id = ClientId, - username = Username, - session = Session, - mountpoint = MountPoint}) -> - TopicTable = parse_topic_table(RawTopicTable), - case emqttd_hooks:run('client.subscribe', [ClientId, Username], TopicTable) of - {ok, TopicTable1} -> - emqttd_session:subscribe(Session, mount(MountPoint, TopicTable1)); - {stop, _} -> - ok - end, - {ok, ProtoState}. - -unsubscribe(RawTopics, ProtoState = #proto_state{client_id = ClientId, - username = Username, - session = Session, - mountpoint = MountPoint}) -> - case emqttd_hooks:run('client.unsubscribe', [ClientId, Username], parse_topics(RawTopics)) of - {ok, TopicTable} -> - emqttd_session:unsubscribe(Session, mount(MountPoint, TopicTable)); - {stop, _} -> - ok - end, - {ok, ProtoState}. - -%% @doc Send PUBREL -pubrel(PacketId, State) -> send(?PUBREL_PACKET(PacketId), State). - -process(?CONNECT_PACKET(Var), State0) -> - - #mqtt_packet_connect{proto_ver = ProtoVer, - proto_name = ProtoName, - username = Username, - password = Password, - clean_sess = CleanSess, - keep_alive = KeepAlive, - client_id = ClientId, - is_bridge = IsBridge} = Var, - - State1 = repl_username_with_peercert( - State0#proto_state{proto_ver = ProtoVer, - proto_name = ProtoName, - username = Username, - client_id = ClientId, - clean_sess = CleanSess, - keepalive = KeepAlive, - will_msg = willmsg(Var, State0), - is_bridge = IsBridge, - connected_at = os:timestamp()}), - - {ReturnCode1, SessPresent, State3} = - case validate_connect(Var, State1) of - ?CONNACK_ACCEPT -> - case authenticate(client(State1), Password) of - {ok, IsSuperuser} -> - %% Generate clientId if null - State2 = maybe_set_clientid(State1), - - %% Start session - case emqttd_sm:start_session(CleanSess, {clientid(State2), Username}) of - {ok, Session, SP} -> - %% Register the client - emqttd_cm:reg(client(State2)), - %% Start keepalive - start_keepalive(KeepAlive, State2), - %% Emit Stats - self() ! emit_stats, - %% ACCEPT - {?CONNACK_ACCEPT, SP, State2#proto_state{session = Session, is_superuser = IsSuperuser}}; - {error, Error} -> - ?LOG(error, "Username '~s' login failed for ~p", [Username, Error], State2), - {?CONNACK_SERVER, false, State2} - end; - {error, Reason}-> - ?LOG(error, "Username '~s' login failed for ~p", [Username, Reason], State1), - {?CONNACK_CREDENTIALS, false, State1} - end; - ReturnCode -> - {ReturnCode, false, State1} - end, - %% Run hooks - emqttd_hooks:run('client.connected', [ReturnCode1], client(State3)), - %% Send connack - send(?CONNACK_PACKET(ReturnCode1, sp(SessPresent)), State3), - %% stop if authentication failure - stop_if_auth_failure(ReturnCode1, State3); - -process(Packet = ?PUBLISH_PACKET(_Qos, Topic, _PacketId, _Payload), State = #proto_state{is_superuser = IsSuper}) -> - case IsSuper orelse allow == check_acl(publish, Topic, client(State)) of - true -> publish(Packet, State); - false -> ?LOG(error, "Cannot publish to ~s for ACL Deny", [Topic], State) - end, - {ok, State}; - -process(?PUBACK_PACKET(?PUBACK, PacketId), State = #proto_state{session = Session}) -> - emqttd_session:puback(Session, PacketId), - {ok, State}; - -process(?PUBACK_PACKET(?PUBREC, PacketId), State = #proto_state{session = Session}) -> - emqttd_session:pubrec(Session, PacketId), - send(?PUBREL_PACKET(PacketId), State); - -process(?PUBACK_PACKET(?PUBREL, PacketId), State = #proto_state{session = Session}) -> - emqttd_session:pubrel(Session, PacketId), - send(?PUBACK_PACKET(?PUBCOMP, PacketId), State); - -process(?PUBACK_PACKET(?PUBCOMP, PacketId), State = #proto_state{session = Session})-> - emqttd_session:pubcomp(Session, PacketId), {ok, State}; - -%% Protect from empty topic table -process(?SUBSCRIBE_PACKET(PacketId, []), State) -> - send(?SUBACK_PACKET(PacketId, []), State); - -%% TODO: refactor later... -process(?SUBSCRIBE_PACKET(PacketId, RawTopicTable), - State = #proto_state{client_id = ClientId, - username = Username, - is_superuser = IsSuperuser, - mountpoint = MountPoint, - session = Session}) -> - Client = client(State), TopicTable = parse_topic_table(RawTopicTable), - AllowDenies = if - IsSuperuser -> []; - true -> [check_acl(subscribe, Topic, Client) || {Topic, _Opts} <- TopicTable] - end, - case lists:member(deny, AllowDenies) of - true -> - ?LOG(error, "Cannot SUBSCRIBE ~p for ACL Deny", [TopicTable], State), - send(?SUBACK_PACKET(PacketId, [16#80 || _ <- TopicTable]), State); - false -> - case emqttd_hooks:run('client.subscribe', [ClientId, Username], TopicTable) of - {ok, TopicTable1} -> - emqttd_session:subscribe(Session, PacketId, mount(MountPoint, TopicTable1)), - {ok, State}; - {stop, _} -> - {ok, State} - end - end; - -%% Protect from empty topic list -process(?UNSUBSCRIBE_PACKET(PacketId, []), State) -> - send(?UNSUBACK_PACKET(PacketId), State); - -process(?UNSUBSCRIBE_PACKET(PacketId, RawTopics), - State = #proto_state{client_id = ClientId, - username = Username, - mountpoint = MountPoint, - session = Session}) -> - case emqttd_hooks:run('client.unsubscribe', [ClientId, Username], parse_topics(RawTopics)) of - {ok, TopicTable} -> - emqttd_session:unsubscribe(Session, mount(MountPoint, TopicTable)); - {stop, _} -> - ok - end, - send(?UNSUBACK_PACKET(PacketId), State); - -process(?PACKET(?PINGREQ), State) -> - send(?PACKET(?PINGRESP), State); - -process(?PACKET(?DISCONNECT), State) -> - % Clean willmsg - {stop, normal, State#proto_state{will_msg = undefined}}. - -publish(Packet = ?PUBLISH_PACKET(?QOS_0, _PacketId), - #proto_state{client_id = ClientId, - username = Username, - mountpoint = MountPoint, - session = Session}) -> - Msg = emqttd_message:from_packet(Username, ClientId, Packet), - emqttd_session:publish(Session, mount(MountPoint, Msg)); - -publish(Packet = ?PUBLISH_PACKET(?QOS_1, _PacketId), State) -> - with_puback(?PUBACK, Packet, State); - -publish(Packet = ?PUBLISH_PACKET(?QOS_2, _PacketId), State) -> - with_puback(?PUBREC, Packet, State). - -with_puback(Type, Packet = ?PUBLISH_PACKET(_Qos, PacketId), - State = #proto_state{client_id = ClientId, - username = Username, - mountpoint = MountPoint, - session = Session}) -> - Msg = emqttd_message:from_packet(Username, ClientId, Packet), - case emqttd_session:publish(Session, mount(MountPoint, Msg)) of - ok -> - send(?PUBACK_PACKET(Type, PacketId), State); - {error, Error} -> - ?LOG(error, "PUBLISH ~p error: ~p", [PacketId, Error], State) - end. - --spec(send(mqtt_message() | mqtt_packet(), proto_state()) -> {ok, proto_state()}). -send(Msg, State = #proto_state{client_id = ClientId, - username = Username, - mountpoint = MountPoint, - is_bridge = IsBridge}) - when is_record(Msg, mqtt_message) -> - emqttd_hooks:run('message.delivered', [ClientId, Username], Msg), - send(emqttd_message:to_packet(unmount(MountPoint, clean_retain(IsBridge, Msg))), State); - -send(Packet = ?PACKET(Type), State = #proto_state{sendfun = SendFun, stats_data = Stats}) -> - trace(send, Packet, State), - emqttd_metrics:sent(Packet), - SendFun(Packet), - {ok, State#proto_state{stats_data = inc_stats(send, Type, Stats)}}. - -trace(recv, Packet, ProtoState) -> - ?LOG(debug, "RECV ~s", [emqttd_packet:format(Packet)], ProtoState); - -trace(send, Packet, ProtoState) -> - ?LOG(debug, "SEND ~s", [emqttd_packet:format(Packet)], ProtoState). - -inc_stats(_Direct, _Type, Stats = #proto_stats{enable_stats = false}) -> - Stats; - -inc_stats(recv, Type, Stats) -> - #proto_stats{recv_pkt = PktCnt, recv_msg = MsgCnt} = Stats, - inc_stats(Type, #proto_stats.recv_pkt, PktCnt, #proto_stats.recv_msg, MsgCnt, Stats); - -inc_stats(send, Type, Stats) -> - #proto_stats{send_pkt = PktCnt, send_msg = MsgCnt} = Stats, - inc_stats(Type, #proto_stats.send_pkt, PktCnt, #proto_stats.send_msg, MsgCnt, Stats). - -inc_stats(Type, PktPos, PktCnt, MsgPos, MsgCnt, Stats) -> - Stats1 = setelement(PktPos, Stats, PktCnt + 1), - case Type =:= ?PUBLISH of - true -> setelement(MsgPos, Stats1, MsgCnt + 1); - false -> Stats1 - end. - -stop_if_auth_failure(RC, State) when RC == ?CONNACK_CREDENTIALS; RC == ?CONNACK_AUTH -> - {stop, {shutdown, auth_failure}, State}; - -stop_if_auth_failure(_RC, State) -> - {ok, State}. - -shutdown(_Error, #proto_state{client_id = undefined}) -> - ignore; -shutdown(conflict, _State) -> - %% let it down - ignore; -shutdown(mnesia_conflict, _State) -> - %% let it down - ignore; -shutdown(Error, State = #proto_state{will_msg = WillMsg}) -> - ?LOG(debug, "Shutdown for ~p", [Error], State), - Client = client(State), - %% Auth failure not publish the will message - case Error =:= auth_failure of - true -> ok; - false -> send_willmsg(State, WillMsg) - end, - emqttd_hooks:run('client.disconnected', [Error], Client), - %% let it down - %% emqttd_cm:unreg(ClientId). - ok. - -willmsg(Packet, #proto_state{mountpoint = MountPoint}) when is_record(Packet, mqtt_packet_connect) -> - case emqttd_message:from_packet(Packet) of - undefined -> undefined; - Msg -> mount(MountPoint, Msg) - end. - -%% Generate a client if if nulll -maybe_set_clientid(State = #proto_state{client_id = NullId}) - when NullId =:= undefined orelse NullId =:= <<>> -> - {_, NPid, _} = emqttd_guid:new(), - ClientId = iolist_to_binary(["emqttd_", integer_to_list(NPid)]), - State#proto_state{client_id = ClientId}; - -maybe_set_clientid(State) -> - State. - -send_willmsg(_State, undefined) -> - ignore; -send_willmsg(State = #proto_state{client_id = ClientId, username = Username, is_superuser = IsSuper}, - WillMsg = #mqtt_message{topic = Topic}) -> - case IsSuper orelse allow == check_acl(publish, Topic, client(State)) of - true -> emqttd:publish(WillMsg#mqtt_message{from = {ClientId, Username}}); - false -> ?LOG(error, "Cannot publish LWT message to ~s for ACL Deny", [Topic], State) - end. - -start_keepalive(0, _State) -> ignore; - -start_keepalive(Sec, #proto_state{keepalive_backoff = Backoff}) when Sec > 0 -> - self() ! {keepalive, start, round(Sec * Backoff)}. - -%%-------------------------------------------------------------------- -%% Validate Packets -%%-------------------------------------------------------------------- - -validate_connect(Connect = #mqtt_packet_connect{}, ProtoState) -> - case validate_protocol(Connect) of - true -> - case validate_clientid(Connect, ProtoState) of - true -> - ?CONNACK_ACCEPT; - false -> - ?CONNACK_INVALID_ID - end; - false -> - ?CONNACK_PROTO_VER - end. - -validate_protocol(#mqtt_packet_connect{proto_ver = Ver, proto_name = Name}) -> - lists:member({Ver, Name}, ?PROTOCOL_NAMES). - -validate_clientid(#mqtt_packet_connect{client_id = ClientId}, - #proto_state{max_clientid_len = MaxLen}) - when (byte_size(ClientId) >= 1) andalso (byte_size(ClientId) =< MaxLen) -> - true; - -%% Issue#599: Null clientId and clean_sess = false -validate_clientid(#mqtt_packet_connect{client_id = ClientId, - clean_sess = CleanSess}, _ProtoState) - when byte_size(ClientId) == 0 andalso (not CleanSess) -> - false; - -%% MQTT3.1.1 allow null clientId. -validate_clientid(#mqtt_packet_connect{proto_ver =?MQTT_PROTO_V4, - client_id = ClientId}, _ProtoState) - when byte_size(ClientId) =:= 0 -> - true; - -validate_clientid(#mqtt_packet_connect{proto_ver = ProtoVer, - clean_sess = CleanSess}, ProtoState) -> - ?LOG(warning, "Invalid clientId. ProtoVer: ~p, CleanSess: ~s", - [ProtoVer, CleanSess], ProtoState), - false. - -validate_packet(?PUBLISH_PACKET(_Qos, Topic, _PacketId, _Payload)) -> - case emqttd_topic:validate({name, Topic}) of - true -> ok; - false -> {error, badtopic} - end; - -validate_packet(?SUBSCRIBE_PACKET(_PacketId, TopicTable)) -> - validate_topics(filter, TopicTable); - -validate_packet(?UNSUBSCRIBE_PACKET(_PacketId, Topics)) -> - validate_topics(filter, Topics); - -validate_packet(_Packet) -> - ok. - -validate_topics(_Type, []) -> - {error, empty_topics}; - -validate_topics(Type, TopicTable = [{_Topic, _Qos}|_]) - when Type =:= name orelse Type =:= filter -> - Valid = fun(Topic, Qos) -> - emqttd_topic:validate({Type, Topic}) and validate_qos(Qos) - end, - case [Topic || {Topic, Qos} <- TopicTable, not Valid(Topic, Qos)] of - [] -> ok; - _ -> {error, badtopic} - end; - -validate_topics(Type, Topics = [Topic0|_]) when is_binary(Topic0) -> - case [Topic || Topic <- Topics, not emqttd_topic:validate({Type, Topic})] of - [] -> ok; - _ -> {error, badtopic} - end. - -validate_qos(undefined) -> - true; -validate_qos(Qos) when ?IS_QOS(Qos) -> - true; -validate_qos(_) -> - false. - -parse_topic_table(TopicTable) -> - lists:map(fun({Topic0, Qos}) -> - {Topic, Opts} = emqttd_topic:parse(Topic0), - {Topic, [{qos, Qos}|Opts]} - end, TopicTable). - -parse_topics(Topics) -> - [emqttd_topic:parse(Topic) || Topic <- Topics]. - -authenticate(Client, Password) -> - case emqttd_access_control:auth(Client, Password) of - ok -> {ok, false}; - {ok, IsSuper} -> {ok, IsSuper}; - {error, Error} -> {error, Error} - end. - -%% PUBLISH ACL is cached in process dictionary. -check_acl(publish, Topic, Client) -> - IfCache = emqttd:env(cache_acl, true), - case {IfCache, get({acl, publish, Topic})} of - {true, undefined} -> - AllowDeny = emqttd_access_control:check_acl(Client, publish, Topic), - put({acl, publish, Topic}, AllowDeny), - AllowDeny; - {true, AllowDeny} -> - AllowDeny; - {false, _} -> - emqttd_access_control:check_acl(Client, publish, Topic) - end; - -check_acl(subscribe, Topic, Client) -> - emqttd_access_control:check_acl(Client, subscribe, Topic). - -sp(true) -> 1; -sp(false) -> 0. - -%%-------------------------------------------------------------------- -%% The retained flag should be propagated for bridge. -%%-------------------------------------------------------------------- - -clean_retain(false, Msg = #mqtt_message{retain = true, headers = Headers}) -> - case lists:member(retained, Headers) of - true -> Msg; - false -> Msg#mqtt_message{retain = false} - end; -clean_retain(_IsBridge, Msg) -> - Msg. - -%%-------------------------------------------------------------------- -%% Mount Point -%%-------------------------------------------------------------------- - -mount(undefined, Any) -> - Any; -mount(MountPoint, Msg = #mqtt_message{topic = Topic}) -> - Msg#mqtt_message{topic = <>}; -mount(MountPoint, TopicTable) when is_list(TopicTable) -> - [{<>, Opts} || {Topic, Opts} <- TopicTable]. - -unmount(undefined, Any) -> - Any; -unmount(MountPoint, Msg = #mqtt_message{topic = Topic}) -> - case catch split_binary(Topic, byte_size(MountPoint)) of - {MountPoint, Topic0} -> Msg#mqtt_message{topic = Topic0}; - _ -> Msg - end. diff --git a/src/emqttd_pubsub.erl b/src/emqttd_pubsub.erl deleted file mode 100644 index 17f5455ff..000000000 --- a/src/emqttd_pubsub.erl +++ /dev/null @@ -1,252 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_pubsub). - --behaviour(gen_server2). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_internal.hrl"). - --export([start_link/3]). - -%% PubSub API. --export([subscribe/3, async_subscribe/3, publish/2, unsubscribe/3, - async_unsubscribe/3, subscribers/1]). - --export([dispatch/2]). - -%% gen_server Callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {pool, id, env}). - --define(PUBSUB, ?MODULE). - --define(is_local(Options), lists:member(local, Options)). - -%%-------------------------------------------------------------------- -%% Start PubSub -%%-------------------------------------------------------------------- - --spec(start_link(atom(), pos_integer(), list()) -> {ok, pid()} | ignore | {error, term()}). -start_link(Pool, Id, Env) -> - gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id, Env], []). - -%%-------------------------------------------------------------------- -%% PubSub API -%%-------------------------------------------------------------------- - -%% @doc Subscribe to a Topic --spec(subscribe(binary(), emqttd:subscriber(), [emqttd:suboption()]) -> ok). -subscribe(Topic, Subscriber, Options) -> - call(pick(Topic), {subscribe, Topic, Subscriber, Options}). - --spec(async_subscribe(binary(), emqttd:subscriber(), [emqttd:suboption()]) -> ok). -async_subscribe(Topic, Subscriber, Options) -> - cast(pick(Topic), {subscribe, Topic, Subscriber, Options}). - -%% @doc Publish MQTT Message to Topic. --spec(publish(binary(), mqtt_message()) -> {ok, mqtt_delivery()} | ignore). -publish(Topic, Msg) -> - route(lists:append(emqttd_router:match(Topic), - emqttd_router:match_local(Topic)), delivery(Msg)). - -route([], #mqtt_delivery{message = #mqtt_message{topic = Topic}}) -> - dropped(Topic), ignore; - -%% Dispatch on the local node. -route([#mqtt_route{topic = To, node = Node}], - Delivery = #mqtt_delivery{flows = Flows}) when Node =:= node() -> - dispatch(To, Delivery#mqtt_delivery{flows = [{route, Node, To} | Flows]}); - -%% Forward to other nodes -route([#mqtt_route{topic = To, node = Node}], Delivery = #mqtt_delivery{flows = Flows}) -> - forward(Node, To, Delivery#mqtt_delivery{flows = [{route, Node, To}|Flows]}); - -route(Routes, Delivery) -> - {ok, lists:foldl(fun(Route, Acc) -> - {ok, Acc1} = route([Route], Acc), Acc1 - end, Delivery, Routes)}. - -delivery(Msg) -> #mqtt_delivery{sender = self(), message = Msg, flows = []}. - -%% @doc Forward message to another node... -forward(Node, To, Delivery) -> - rpc:cast(Node, ?PUBSUB, dispatch, [To, Delivery]), {ok, Delivery}. - -%% @doc Dispatch Message to Subscribers. --spec(dispatch(binary(), mqtt_delivery()) -> mqtt_delivery()). -dispatch(Topic, Delivery = #mqtt_delivery{message = Msg, flows = Flows}) -> - case subscribers(Topic) of - [] -> - dropped(Topic), {ok, Delivery}; - [Sub] -> %% optimize? - dispatch(Sub, Topic, Msg), - {ok, Delivery#mqtt_delivery{flows = [{dispatch, Topic, 1}|Flows]}}; - Subscribers -> - Flows1 = [{dispatch, Topic, length(Subscribers)} | Flows], - lists:foreach(fun(Sub) -> dispatch(Sub, Topic, Msg) end, Subscribers), - {ok, Delivery#mqtt_delivery{flows = Flows1}} - end. - -%%TODO: Is SubPid aliving??? -dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> - SubPid ! {dispatch, Topic, Msg}; -dispatch({SubId, SubPid}, Topic, Msg) when is_binary(SubId), is_pid(SubPid) -> - SubPid ! {dispatch, Topic, Msg}; -dispatch({{share, _Share}, [Sub]}, Topic, Msg) -> - dispatch(Sub, Topic, Msg); -dispatch({{share, _Share}, []}, _Topic, _Msg) -> - ok; -dispatch({{share, _Share}, Subs}, Topic, Msg) -> %% round-robbin? - dispatch(lists:nth(rand:uniform(length(Subs)), Subs), Topic, Msg). - -subscribers(Topic) -> - group_by_share(try ets:lookup_element(mqtt_subscriber, Topic, 2) catch error:badarg -> [] end). - -group_by_share([]) -> []; - -group_by_share(Subscribers) -> - {Subs1, Shares1} = - lists:foldl(fun({share, Share, Sub}, {Subs, Shares}) -> - {Subs, dict:append({share, Share}, Sub, Shares)}; - (Sub, {Subs, Shares}) -> - {[Sub|Subs], Shares} - end, {[], dict:new()}, Subscribers), - lists:append(Subs1, dict:to_list(Shares1)). - -%% @private -%% @doc Ingore $SYS Messages. -dropped(<<"$SYS/", _/binary>>) -> - ok; -dropped(_Topic) -> - emqttd_metrics:inc('messages/dropped'). - -%% @doc Unsubscribe --spec(unsubscribe(binary(), emqttd:subscriber(), [emqttd:suboption()]) -> ok). -unsubscribe(Topic, Subscriber, Options) -> - call(pick(Topic), {unsubscribe, Topic, Subscriber, Options}). - --spec(async_unsubscribe(binary(), emqttd:subscriber(), [emqttd:suboption()]) -> ok). -async_unsubscribe(Topic, Subscriber, Options) -> - cast(pick(Topic), {unsubscribe, Topic, Subscriber, Options}). - -call(PubSub, Req) when is_pid(PubSub) -> - gen_server2:call(PubSub, Req, infinity). - -cast(PubSub, Msg) when is_pid(PubSub) -> - gen_server2:cast(PubSub, Msg). - -pick(Topic) -> - gproc_pool:pick_worker(pubsub, Topic). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id, Env]) -> - ?GPROC_POOL(join, Pool, Id), - {ok, #state{pool = Pool, id = Id, env = Env}, - hibernate, {backoff, 2000, 2000, 20000}}. - -handle_call({subscribe, Topic, Subscriber, Options}, _From, State) -> - add_subscriber(Topic, Subscriber, Options), - reply(ok, setstats(State)); - -handle_call({unsubscribe, Topic, Subscriber, Options}, _From, State) -> - del_subscriber(Topic, Subscriber, Options), - reply(ok, setstats(State)); - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast({subscribe, Topic, Subscriber, Options}, State) -> - add_subscriber(Topic, Subscriber, Options), - noreply(setstats(State)); - -handle_cast({unsubscribe, Topic, Subscriber, Options}, State) -> - del_subscriber(Topic, Subscriber, Options), - noreply(setstats(State)); - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - ?GPROC_POOL(leave, Pool, Id). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internel Functions -%%-------------------------------------------------------------------- - -add_subscriber(Topic, Subscriber, Options) -> - Share = proplists:get_value(share, Options), - case ?is_local(Options) of - false -> add_global_subscriber(Share, Topic, Subscriber); - true -> add_local_subscriber(Share, Topic, Subscriber) - end. - -add_global_subscriber(Share, Topic, Subscriber) -> - case ets:member(mqtt_subscriber, Topic) and emqttd_router:has_route(Topic) of - true -> ok; - false -> emqttd_router:add_route(Topic) - end, - ets:insert(mqtt_subscriber, {Topic, shared(Share, Subscriber)}). - -add_local_subscriber(Share, Topic, Subscriber) -> - (not ets:member(mqtt_subscriber, {local, Topic})) andalso emqttd_router:add_local_route(Topic), - ets:insert(mqtt_subscriber, {{local, Topic}, shared(Share, Subscriber)}). - -del_subscriber(Topic, Subscriber, Options) -> - Share = proplists:get_value(share, Options), - case ?is_local(Options) of - false -> del_global_subscriber(Share, Topic, Subscriber); - true -> del_local_subscriber(Share, Topic, Subscriber) - end. - -del_global_subscriber(Share, Topic, Subscriber) -> - ets:delete_object(mqtt_subscriber, {Topic, shared(Share, Subscriber)}), - (not ets:member(mqtt_subscriber, Topic)) andalso emqttd_router:del_route(Topic). - -del_local_subscriber(Share, Topic, Subscriber) -> - ets:delete_object(mqtt_subscriber, {{local, Topic}, shared(Share, Subscriber)}), - (not ets:member(mqtt_subscriber, {local, Topic})) andalso emqttd_router:del_local_route(Topic). - -shared(undefined, Subscriber) -> - Subscriber; -shared(Share, Subscriber) -> - {share, Share, Subscriber}. - -setstats(State) -> - emqttd_stats:setstats('subscribers/count', 'subscribers/max', ets:info(mqtt_subscriber, size)), - State. - -reply(Reply, State) -> - {reply, Reply, State, hibernate}. - -noreply(State) -> - {noreply, State, hibernate}. - diff --git a/src/emqttd_pubsub_sup.erl b/src/emqttd_pubsub_sup.erl deleted file mode 100644 index 6e18aa031..000000000 --- a/src/emqttd_pubsub_sup.erl +++ /dev/null @@ -1,86 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc PubSub Supervisor. --module(emqttd_pubsub_sup). - --author("Feng Lee "). - --behaviour(supervisor). - -%% API --export([start_link/0, pubsub_pool/0]). - -%% Supervisor callbacks --export([init/1]). - --define(CONCURRENCY_OPTS, [{read_concurrency, true}, {write_concurrency, true}]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -pubsub_pool() -> - hd([Pid || {pubsub_pool, Pid, _, _} <- supervisor:which_children(?MODULE)]). - -%%-------------------------------------------------------------------- -%% Supervisor Callbacks -%%-------------------------------------------------------------------- - -init([]) -> - {ok, Env} = emqttd:env(pubsub), - %% Create ETS Tables - [create_tab(Tab) || Tab <- [mqtt_subproperty, mqtt_subscriber, mqtt_subscription]], - {ok, { {one_for_all, 10, 3600}, [pool_sup(pubsub, Env), pool_sup(server, Env)]} }. - -%%-------------------------------------------------------------------- -%% Pool -%%-------------------------------------------------------------------- - -pool_size(Env) -> - Schedulers = erlang:system_info(schedulers), - proplists:get_value(pool_size, Env, Schedulers). - -pool_sup(Name, Env) -> - Pool = list_to_atom(atom_to_list(Name) ++ "_pool"), - Mod = list_to_atom("emqttd_" ++ atom_to_list(Name)), - MFA = {Mod, start_link, [Env]}, - emqttd_pool_sup:spec(Pool, [Name, hash, pool_size(Env), MFA]). - -%%-------------------------------------------------------------------- -%% Create PubSub Tables -%%-------------------------------------------------------------------- - -create_tab(mqtt_subproperty) -> - %% Subproperty: {Topic, Sub} -> [{qos, 1}] - ensure_tab(mqtt_subproperty, [public, named_table, set | ?CONCURRENCY_OPTS]); - -create_tab(mqtt_subscriber) -> - %% Subscriber: Topic -> Sub1, Sub2, Sub3, ..., SubN - %% duplicate_bag: o(1) insert - ensure_tab(mqtt_subscriber, [public, named_table, duplicate_bag | ?CONCURRENCY_OPTS]); - -create_tab(mqtt_subscription) -> - %% Subscription: Sub -> Topic1, Topic2, Topic3, ..., TopicN - %% bag: o(n) insert - ensure_tab(mqtt_subscription, [public, named_table, bag | ?CONCURRENCY_OPTS]). - -ensure_tab(Tab, Opts) -> - case ets:info(Tab, name) of undefined -> ets:new(Tab, Opts); _ -> ok end. - diff --git a/src/emqttd_rest_api.erl b/src/emqttd_rest_api.erl deleted file mode 100644 index 1e4de392c..000000000 --- a/src/emqttd_rest_api.erl +++ /dev/null @@ -1,550 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module (emqttd_rest_api). - --include("emqttd.hrl"). - --include("emqttd_internal.hrl"). - --http_api({"^nodes/(.+?)/alarms/?$", 'GET', alarm_list, []}). - --http_api({"^nodes/(.+?)/clients/?$", 'GET', client_list, []}). --http_api({"^nodes/(.+?)/clients/(.+?)/?$", 'GET',client_list, []}). --http_api({"^clients/(.+?)/?$", 'GET', client, []}). --http_api({"^clients/(.+?)/?$", 'DELETE', kick_client, []}). --http_api({"^clients/(.+?)/clean_acl_cache?$", 'PUT', clean_acl_cache, [{<<"topic">>, binary}]}). - --http_api({"^routes?$", 'GET', route_list, []}). --http_api({"^routes/(.+?)/?$", 'GET', route, []}). - --http_api({"^nodes/(.+?)/sessions/?$", 'GET', session_list, []}). --http_api({"^nodes/(.+?)/sessions/(.+?)/?$", 'GET', session_list, []}). --http_api({"^sessions/(.+?)/?$", 'GET', session, []}). - --http_api({"^nodes/(.+?)/subscriptions/?$", 'GET', subscription_list, []}). --http_api({"^nodes/(.+?)/subscriptions/(.+?)/?$", 'GET', subscription_list, []}). --http_api({"^subscriptions/(.+?)/?$", 'GET', subscription, []}). - --http_api({"^mqtt/publish?$", 'POST', publish, [{<<"topic">>, binary}, {<<"payload">>, binary}]}). --http_api({"^mqtt/subscribe?$", 'POST', subscribe, [{<<"client_id">>, binary},{<<"topic">>, binary}]}). --http_api({"^mqtt/unsubscribe?$", 'POST', unsubscribe, [{<<"client_id">>, binary},{<<"topic">>, binary}]}). - --http_api({"^management/nodes/?$", 'GET', brokers, []}). --http_api({"^management/nodes/(.+?)/?$", 'GET', broker, []}). --http_api({"^monitoring/nodes/?$", 'GET', nodes, []}). --http_api({"^monitoring/nodes/(.+?)/?$", 'GET', node, []}). --http_api({"^monitoring/listeners/?$", 'GET', listeners, []}). --http_api({"^monitoring/listeners/(.+?)/?$", 'GET', listener, []}). --http_api({"^monitoring/metrics/?$", 'GET', metrics, []}). --http_api({"^monitoring/metrics/(.+?)/?$", 'GET', metric, []}). --http_api({"^monitoring/stats/?$", 'GET', stats, []}). --http_api({"^monitoring/stats/(.+?)/?$", 'GET', stat, []}). - --http_api({"^nodes/(.+?)/plugins/?$", 'GET', plugin_list, []}). --http_api({"^nodes/(.+?)/plugins/(.+?)/?$", 'PUT', enabled, [{<<"active">>, bool}]}). - --http_api({"^configs/(.+?)/?$", 'PUT', modify_config, [{<<"key">>, binary}, {<<"value">>, binary}]}). --http_api({"^configs/?$", 'GET', config_list, []}). --http_api({"^nodes/(.+?)/configs/(.+?)/?$", 'PUT', modify_config, [{<<"key">>, binary}, {<<"value">>, binary}]}). --http_api({"^nodes/(.+?)/configs/?$", 'GET', config_list, []}). --http_api({"^nodes/(.+?)/plugin_configs/(.+?)/?$", 'GET', plugin_config_list, []}). --http_api({"^nodes/(.+?)/plugin_configs/(.+?)/?$", 'PUT', modify_plugin_config, []}). - --http_api({"^users/?$", 'GET', users, []}). --http_api({"^users/?$", 'POST', users, [{<<"username">>, binary}, - {<<"password">>, binary}, - {<<"tags">>, binary}]}). --http_api({"^users/(.+?)/?$", 'GET', users, []}). --http_api({"^users/(.+?)/?$", 'PUT', users, [{<<"tags">>, binary}]}). --http_api({"^users/(.+?)/?$", 'DELETE', users, []}). - --http_api({"^auth/?$", 'POST', auth, [{<<"username">>, binary}, {<<"password">>, binary}]}). --http_api({"^change_pwd/(.+?)/?$", 'PUT', change_pwd, [{<<"old_pwd">>, binary}, - {<<"new_pwd">>, binary}]}). - --import(proplists, [get_value/2, get_value/3]). - --export([alarm_list/3]). --export([client/3, client_list/3, client_list/4, kick_client/3, clean_acl_cache/3]). --export([route/3, route_list/2]). --export([session/3, session_list/3, session_list/4]). --export([subscription/3, subscription_list/3, subscription_list/4]). --export([nodes/2, node/3, brokers/2, broker/3, listeners/2, listener/3, metrics/2, metric/3, stats/2, stat/3]). --export([publish/2, subscribe/2, unsubscribe/2]). --export([plugin_list/3, enabled/4]). --export([modify_config/3, modify_config/4, config_list/2, config_list/3, - plugin_config_list/4, modify_plugin_config/4]). - --export([users/2,users/3, auth/2, change_pwd/3]). - -%%-------------------------------------------------------------------------- -%% alarm -%%-------------------------------------------------------------------------- -alarm_list('GET', _Req, _Node) -> - Alarms = emqttd_mgmt:alarm_list(), - {ok, lists:map(fun alarm_row/1, Alarms)}. - -alarm_row(#mqtt_alarm{id = AlarmId, - severity = Severity, - title = Title, - summary = Summary, - timestamp = Timestamp}) -> - [{id, AlarmId}, - {severity, Severity}, - {title, l2b(Title)}, - {summary, l2b(Summary)}, - {occurred_at, l2b(strftime(Timestamp))}]. - -%%-------------------------------------------------------------------------- -%% client -%%-------------------------------------------------------------------------- -client('GET', _Params, Key) -> - Data = emqttd_mgmt:client(l2b(Key)), - {ok, [{objects, [client_row(Row) || Row <- Data]}]}. - -client_list('GET', Params, Node) -> - {PageNo, PageSize} = page_params(Params), - Data = emqttd_mgmt:client_list(l2a(Node), undefined, PageNo, PageSize), - Rows = get_value(result, Data), - TotalPage = get_value(totalPage, Data), - TotalNum = get_value(totalNum, Data), - {ok, [{current_page, PageNo}, - {page_size, PageSize}, - {total_num, TotalNum}, - {total_page, TotalPage}, - {objects, [client_row(Row) || Row <- Rows]}]}. - -client_list('GET', Params, Node, Key) -> - {PageNo, PageSize} = page_params(Params), - Data = emqttd_mgmt:client_list(l2a(Node), l2b(Key), PageNo, PageSize), - {ok, [{objects, [client_row(Row) || Row <- Data]}]}. - -kick_client('DELETE', _Params, Key) -> - case emqttd_mgmt:kick_client(l2b(Key)) of - true -> {ok, []}; - false -> {error, [{code, ?ERROR12}]} - end. - -clean_acl_cache('PUT', Params, Key0) -> - Topic = get_value(<<"topic">>, Params), - [Key | _] = string:tokens(Key0, "/"), - case emqttd_mgmt:clean_acl_cache(l2b(Key), Topic) of - true -> {ok, []}; - false -> {error, [{code, ?ERROR12}]} - end. - -client_row(#mqtt_client{client_id = ClientId, - peername = {IpAddr, Port}, - username = Username, - clean_sess = CleanSess, - proto_ver = ProtoVer, - keepalive = KeepAlvie, - connected_at = ConnectedAt}) -> - [{client_id, ClientId}, - {username, Username}, - {ipaddress, l2b(ntoa(IpAddr))}, - {port, Port}, - {clean_sess, CleanSess}, - {proto_ver, ProtoVer}, - {keepalive, KeepAlvie}, - {connected_at, l2b(strftime(ConnectedAt))}]. - -%%-------------------------------------------------------------------------- -%% route -%%-------------------------------------------------------------------------- -route('GET', _Params, Key) -> - Data = emqttd_mgmt:route(l2b(Key)), - {ok, [{objects, [route_row(Row) || Row <- Data]}]}. - -route_list('GET', Params) -> - {PageNo, PageSize} = page_params(Params), - Data = emqttd_mgmt:route_list(undefined, PageNo, PageSize), - Rows = get_value(result, Data), - TotalPage = get_value(totalPage, Data), - TotalNum = get_value(totalNum, Data), - {ok, [{current_page, PageNo}, - {page_size, PageSize}, - {total_num, TotalNum}, - {total_page, TotalPage}, - {objects, [route_row(Row) || Row <- Rows]}]}. - -route_row(Route) when is_record(Route, mqtt_route) -> - [{topic, Route#mqtt_route.topic}, {node, Route#mqtt_route.node}]; - -route_row({Topic, Node}) -> - [{topic, Topic}, {node, Node}]. - -%%-------------------------------------------------------------------------- -%% session -%%-------------------------------------------------------------------------- -session('GET', _Params, Key) -> - Data = emqttd_mgmt:session(l2b(Key)), - {ok, [{objects, [session_row(Row) || Row <- Data]}]}. - -session_list('GET', Params, Node) -> - {PageNo, PageSize} = page_params(Params), - Data = emqttd_mgmt:session_list(l2a(Node), undefined, PageNo, PageSize), - Rows = get_value(result, Data), - TotalPage = get_value(totalPage, Data), - TotalNum = get_value(totalNum, Data), - {ok, [{current_page, PageNo}, - {page_size, PageSize}, - {total_num, TotalNum}, - {total_page, TotalPage}, - {objects, [session_row(Row) || Row <- Rows]}]}. - -session_list('GET', Params, Node, ClientId) -> - {PageNo, PageSize} = page_params(Params), - Data = emqttd_mgmt:session_list(l2a(Node), l2b(ClientId), PageNo, PageSize), - {ok, [{objects, [session_row(Row) || Row <- Data]}]}. - -session_row({ClientId, _Pid, _Persistent, Session}) -> - Data = lists:append(Session, emqttd_stats:get_session_stats(ClientId)), - InfoKeys = [clean_sess, subscriptions, max_inflight, inflight_len, mqueue_len, - mqueue_dropped, awaiting_rel_len, deliver_msg,enqueue_msg, created_at], - [{client_id, ClientId} | [{Key, format(Key, get_value(Key, Data))} || Key <- InfoKeys]]. - -%%-------------------------------------------------------------------------- -%% subscription -%%-------------------------------------------------------------------------- -subscription('GET', _Params, Key) -> - Data = emqttd_mgmt:subscription(l2b(Key)), - {ok, [{objects, [subscription_row(Row) || Row <- Data]}]}. - -subscription_list('GET', Params, Node) -> - {PageNo, PageSize} = page_params(Params), - Data = emqttd_mgmt:subscription_list(l2a(Node), undefined, PageNo, PageSize), - Rows = get_value(result, Data), - TotalPage = get_value(totalPage, Data), - TotalNum = get_value(totalNum, Data), - {ok, [{current_page, PageNo}, - {page_size, PageSize}, - {total_num, TotalNum}, - {total_page, TotalPage}, - {objects, [subscription_row(Row) || Row <- Rows]}]}. - -subscription_list('GET', Params, Node, Key) -> - {PageNo, PageSize} = page_params(Params), - Data = emqttd_mgmt:subscription_list(l2a(Node), l2b(Key), PageNo, PageSize), - {ok, [{objects, [subscription_row(Row) || Row <- Data]}]}. - -subscription_row({{Topic, SubPid}, Options}) when is_pid(SubPid) -> - subscription_row({{Topic, {undefined, SubPid}}, Options}); -subscription_row({{Topic, {SubId, SubPid}}, Options}) -> - Qos = proplists:get_value(qos, Options), - ClientId = case SubId of - undefined -> list_to_binary(pid_to_list(SubPid)); - SubId -> SubId - end, - [{client_id, ClientId}, {topic, Topic}, {qos, Qos}]. - -%%-------------------------------------------------------------------------- -%% management/monitoring -%%-------------------------------------------------------------------------- -nodes('GET', _Params) -> - Data = emqttd_mgmt:nodes_info(), - {ok, Data}. - -node('GET', _Params, Node) -> - Data = emqttd_mgmt:node_info(l2a(Node)), - {ok, Data}. - -brokers('GET', _Params) -> - Data = emqttd_mgmt:brokers(), - {ok, [format_broker(Node, Broker) || {Node, Broker} <- Data]}. - -broker('GET', _Params, Node) -> - Data = emqttd_mgmt:broker(l2a(Node)), - {ok, format_broker(Data)}. - -listeners('GET', _Params) -> - Data = emqttd_mgmt:listeners(), - {ok, [[{Node, format_listeners(Listeners, [])} || {Node, Listeners} <- Data]]}. - -listener('GET', _Params, Node) -> - Data = emqttd_mgmt:listener(l2a(Node)), - {ok, [format_listener(Listeners) || Listeners <- Data]}. - -metrics('GET', _Params) -> - Data = emqttd_mgmt:metrics(), - {ok, [Data]}. - -metric('GET', _Params, Node) -> - Data = emqttd_mgmt:metrics(l2a(Node)), - {ok, Data}. - -stats('GET', _Params) -> - Data = emqttd_mgmt:stats(), - {ok, [Data]}. - -stat('GET', _Params, Node) -> - Data = emqttd_mgmt:stats(l2a(Node)), - {ok, Data}. - -format_broker(Node, Broker) -> - OtpRel = "R" ++ erlang:system_info(otp_release) ++ "/" ++ erlang:system_info(version), - [{name, Node}, - {version, bin(get_value(version, Broker))}, - {sysdescr, bin(get_value(sysdescr, Broker))}, - {uptime, bin(get_value(uptime, Broker))}, - {datetime, bin(get_value(datetime, Broker))}, - {otp_release, l2b(OtpRel)}, - {node_status, 'Running'}]. - -format_broker(Broker) -> - OtpRel = "R" ++ erlang:system_info(otp_release) ++ "/" ++ erlang:system_info(version), - [{version, bin(get_value(version, Broker))}, - {sysdescr, bin(get_value(sysdescr, Broker))}, - {uptime, bin(get_value(uptime, Broker))}, - {datetime, bin(get_value(datetime, Broker))}, - {otp_release, l2b(OtpRel)}, - {node_status, 'Running'}]. - -format_listeners([], Acc) -> - Acc; -format_listeners([{Protocol, ListenOn, Info}| Listeners], Acc) -> - format_listeners(Listeners, [format_listener({Protocol, ListenOn, Info}) | Acc]). - -format_listener({Protocol, ListenOn, Info}) -> - Listen = l2b(esockd:to_string(ListenOn)), - lists:append([{protocol, Protocol}, {listen, Listen}], Info). - -%%-------------------------------------------------------------------------- -%% mqtt -%%-------------------------------------------------------------------------- -publish('POST', Params) -> - Topic = get_value(<<"topic">>, Params), - ClientId = get_value(<<"client_id">>, Params, http), - Payload = get_value(<<"payload">>, Params, <<>>), - Qos = get_value(<<"qos">>, Params, 0), - Retain = get_value(<<"retain">>, Params, false), - case emqttd_mgmt:publish({ClientId, Topic, Payload, Qos, Retain}) of - ok -> - {ok, []}; - {error, Error} -> - {error, [{code, ?ERROR2}, {message, Error}]} - end. - -subscribe('POST', Params) -> - ClientId = get_value(<<"client_id">>, Params), - Topic = get_value(<<"topic">>, Params), - Qos = get_value(<<"qos">>, Params, 0), - case emqttd_mgmt:subscribe({ClientId, Topic, Qos}) of - ok -> - {ok, []}; - {error, Error} -> - {error, [{code, ?ERROR2}, {message, Error}]} - end. - -unsubscribe('POST', Params) -> - ClientId = get_value(<<"client_id">>, Params), - Topic = get_value(<<"topic">>, Params), - case emqttd_mgmt:unsubscribe({ClientId, Topic})of - ok -> - {ok, []}; - {error, Error} -> - {error, [{code, ?ERROR2}, {message, Error}]} - end. - -%%-------------------------------------------------------------------------- -%% plugins -%%-------------------------------------------------------------------------- -plugin_list('GET', _Params, Node) -> - Plugins = lists:map(fun plugin/1, emqttd_mgmt:plugin_list(l2a(Node))), - {ok, Plugins}. - -enabled('PUT', Params, Node, PluginName) -> - Active = get_value(<<"active">>, Params), - case Active of - true -> - return(emqttd_mgmt:plugin_load(l2a(Node), l2a(PluginName))); - false -> - return(emqttd_mgmt:plugin_unload(l2a(Node), l2a(PluginName))) - end. - -return(Result) -> - case Result of - ok -> - {ok, []}; - {ok, _} -> - {ok, []}; - {error, already_started} -> - {error, [{code, ?ERROR10}, {message, <<"already_started">>}]}; - {error, not_started} -> - {error, [{code, ?ERROR11}, {message, <<"not_started">>}]}; - Error -> - lager:error("error:~p", [Error]), - {error, [{code, ?ERROR2}, {message, <<"unknown">>}]} - end. -plugin(#mqtt_plugin{name = Name, version = Ver, descr = Descr, - active = Active}) -> - [{name, Name}, - {version, iolist_to_binary(Ver)}, - {description, iolist_to_binary(Descr)}, - {active, Active}]. - -%%-------------------------------------------------------------------------- -%% modify config -%%-------------------------------------------------------------------------- -modify_config('PUT', Params, App) -> - Key = get_value(<<"key">>, Params, <<"">>), - Value = get_value(<<"value">>, Params, <<"">>), - case emqttd_mgmt:modify_config(l2a(App), b2l(Key), b2l(Value)) of - true -> {ok, []}; - false -> {error, [{code, ?ERROR2}]} - end. - -modify_config('PUT', Params, Node, App) -> - Key = get_value(<<"key">>, Params, <<"">>), - Value = get_value(<<"value">>, Params, <<"">>), - case emqttd_mgmt:modify_config(l2a(Node), l2a(App), b2l(Key), b2l(Value)) of - ok -> {ok, []}; - _ -> {error, [{code, ?ERROR2}]} - end. - -config_list('GET', _Params) -> - Data = emqttd_mgmt:get_configs(), - {ok, [{Node, format_config(Config, [])} || {Node, Config} <- Data]}. - -config_list('GET', _Params, Node) -> - Data = emqttd_mgmt:get_config(l2a(Node)), - {ok, [format_config(Config) || Config <- lists:reverse(Data)]}. - -plugin_config_list('GET', _Params, Node, App) -> - {ok, Data} = emqttd_mgmt:get_plugin_config(l2a(Node), l2a(App)), - {ok, [format_plugin_config(Config) || Config <- lists:reverse(Data)]}. - -modify_plugin_config('PUT', Params, Node, App) -> - PluginName = l2a(App), - case emqttd_mgmt:modify_plugin_config(l2a(Node), PluginName, Params) of - ok -> - Plugins = emqttd_plugins:list(), - {_, _, _, _, Status} = lists:keyfind(PluginName, 2, Plugins), - case Status of - true -> - emqttd_plugins:unload(PluginName), - timer:sleep(500), - emqttd_plugins:load(PluginName), - {ok, []}; - false -> - {ok, []} - end; - _ -> - {error, [{code, ?ERROR2}]} - end. - - -format_config([], Acc) -> - Acc; -format_config([{Key, Value, Datatpye, App}| Configs], Acc) -> - format_config(Configs, [format_config({Key, Value, Datatpye, App}) | Acc]). - -format_config({Key, Value, Datatpye, App}) -> - [{<<"key">>, l2b(Key)}, - {<<"value">>, l2b(Value)}, - {<<"datatpye">>, l2b(Datatpye)}, - {<<"app">>, App}]. - -format_plugin_config({Key, Value, Desc, Required}) -> - [{<<"key">>, l2b(Key)}, - {<<"value">>, l2b(Value)}, - {<<"desc">>, l2b(Desc)}, - {<<"required">>, Required}]. - -%%-------------------------------------------------------------------------- -%% Admin -%%-------------------------------------------------------------------------- -auth('POST', Params) -> - Username = get_value(<<"username">>, Params), - Password = get_value(<<"password">>, Params), - case emqttd_mgmt:check_user(Username, Password) of - ok -> - {ok, []}; - {error, Reason} -> - {error, [{code, ?ERROR3}, {message, list_to_binary(Reason)}]} - end. - -users('POST', Params) -> - Username = get_value(<<"username">>, Params), - Password = get_value(<<"password">>, Params), - Tag = get_value(<<"tags">>, Params), - code(emqttd_mgmt:add_user(Username, Password, Tag)); - -users('GET', _Params) -> - {ok, [Admin || Admin <- emqttd_mgmt:user_list()]}. - -users('GET', _Params, Username) -> - {ok, emqttd_mgmt:lookup_user(list_to_binary(Username))}; - -users('PUT', Params, Username) -> - code(emqttd_mgmt:update_user(list_to_binary(Username), Params)); - -users('DELETE', _Params, "admin") -> - {error, [{code, ?ERROR6}, {message, <<"admin cannot be deleted">>}]}; -users('DELETE', _Params, Username) -> - code(emqttd_mgmt:remove_user(list_to_binary(Username))). - -change_pwd('PUT', Params, Username) -> - OldPwd = get_value(<<"old_pwd">>, Params), - NewPwd = get_value(<<"new_pwd">>, Params), - code(emqttd_mgmt:change_password(list_to_binary(Username), OldPwd, NewPwd)). - -code(ok) -> {ok, []}; -code(error) -> {error, [{code, ?ERROR2}]}; -code({error, Error}) -> {error, Error}. -%%-------------------------------------------------------------------------- -%% Inner function -%%-------------------------------------------------------------------------- -format(created_at, Val) -> - l2b(strftime(Val)); -format(_, Val) -> - Val. - -ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> - inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}); -ntoa(IP) -> - inet_parse:ntoa(IP). - -%%-------------------------------------------------------------------- -%% Strftime -%%-------------------------------------------------------------------- -strftime({MegaSecs, Secs, _MicroSecs}) -> - strftime(datetime(MegaSecs * 1000000 + Secs)); - -strftime({{Y,M,D}, {H,MM,S}}) -> - lists:flatten( - io_lib:format( - "~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S])). - -datetime(Timestamp) when is_integer(Timestamp) -> - Universal = calendar:gregorian_seconds_to_datetime(Timestamp + - calendar:datetime_to_gregorian_seconds({{1970,1,1}, {0,0,0}})), - calendar:universal_time_to_local_time(Universal). - -bin(S) when is_list(S) -> l2b(S); -bin(A) when is_atom(A) -> bin(atom_to_list(A)); -bin(B) when is_binary(B) -> B; -bin(undefined) -> <<>>. -int(L) -> list_to_integer(L). -l2a(L) -> list_to_atom(L). -l2b(L) -> list_to_binary(L). -b2l(B) -> binary_to_list(B). - - -page_params(Params) -> - PageNo = int(get_value("curr_page", Params, "1")), - PageSize = int(get_value("page_size", Params, "20")), - {PageNo, PageSize}. diff --git a/src/emqttd_router.erl b/src/emqttd_router.erl deleted file mode 100644 index f667f8ea0..000000000 --- a/src/emqttd_router.erl +++ /dev/null @@ -1,290 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_router). - --author("Feng Lee "). - --behaviour(gen_server). - --include("emqttd.hrl"). - -%% Mnesia Bootstrap --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - --export([start_link/0, topics/0, local_topics/0]). - -%% For eunit tests --export([start/0, stop/0]). - -%% Route APIs --export([add_route/1, del_route/1, match/1, print/1, has_route/1]). - -%% Local Route API --export([get_local_routes/0, add_local_route/1, match_local/1, - del_local_route/1, clean_local_routes/0]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --export([dump/0]). - --record(state, {stats_timer}). - --define(ROUTER, ?MODULE). - --define(LOCK, {?ROUTER, clean_routes}). - -%%-------------------------------------------------------------------- -%% Mnesia Bootstrap -%%-------------------------------------------------------------------- - -mnesia(boot) -> - ok = ekka_mnesia:create_table(mqtt_route, [ - {type, bag}, - {ram_copies, [node()]}, - {record_name, mqtt_route}, - {attributes, record_info(fields, mqtt_route)}]); - -mnesia(copy) -> - ok = ekka_mnesia:copy_table(mqtt_route, ram_copies). - -%%-------------------------------------------------------------------- -%% Start the Router -%%-------------------------------------------------------------------- - -start_link() -> - gen_server:start_link({local, ?ROUTER}, ?MODULE, [], []). - -%%-------------------------------------------------------------------- -%% Topics -%%-------------------------------------------------------------------- - --spec(topics() -> list(binary())). -topics() -> - mnesia:dirty_all_keys(mqtt_route). - --spec(local_topics() -> list(binary())). -local_topics() -> - ets:select(mqtt_local_route, [{{'$1', '_'}, [], ['$1']}]). - -%%-------------------------------------------------------------------- -%% Match API -%%-------------------------------------------------------------------- - -%% @doc Match Routes. --spec(match(Topic:: binary()) -> [mqtt_route()]). -match(Topic) when is_binary(Topic) -> - %% Optimize: ets??? - Matched = mnesia:ets(fun emqttd_trie:match/1, [Topic]), - %% Optimize: route table will be replicated to all nodes. - lists:append([ets:lookup(mqtt_route, To) || To <- [Topic | Matched]]). - -%% @doc Print Routes. --spec(print(Topic :: binary()) -> [ok]). -print(Topic) -> - [io:format("~s -> ~s~n", [To, Node]) || - #mqtt_route{topic = To, node = Node} <- match(Topic)]. - -%%-------------------------------------------------------------------- -%% Route Management API -%%-------------------------------------------------------------------- - -%% @doc Add Route. --spec(add_route(binary() | mqtt_route()) -> ok | {error, Reason :: term()}). -add_route(Topic) when is_binary(Topic) -> - add_route(#mqtt_route{topic = Topic, node = node()}); -add_route(Route = #mqtt_route{topic = Topic}) -> - case emqttd_topic:wildcard(Topic) of - true -> case mnesia:is_transaction() of - true -> add_trie_route(Route); - false -> trans(fun add_trie_route/1, [Route]) - end; - false -> add_direct_route(Route) - end. - -add_direct_route(Route) -> - mnesia:async_dirty(fun mnesia:write/1, [Route]). - -add_trie_route(Route = #mqtt_route{topic = Topic}) -> - case mnesia:wread({mqtt_route, Topic}) of - [] -> emqttd_trie:insert(Topic); - _ -> ok - end, - mnesia:write(Route). - -%% @doc Delete Route --spec(del_route(binary() | mqtt_route()) -> ok | {error, Reason :: term()}). -del_route(Topic) when is_binary(Topic) -> - del_route(#mqtt_route{topic = Topic, node = node()}); -del_route(Route = #mqtt_route{topic = Topic}) -> - case emqttd_topic:wildcard(Topic) of - true -> case mnesia:is_transaction() of - true -> del_trie_route(Route); - false -> trans(fun del_trie_route/1, [Route]) - end; - false -> del_direct_route(Route) - end. - -del_direct_route(Route) -> - mnesia:async_dirty(fun mnesia:delete_object/1, [Route]). - -del_trie_route(Route = #mqtt_route{topic = Topic}) -> - case mnesia:wread({mqtt_route, Topic}) of - [Route] -> %% Remove route and trie - mnesia:delete_object(Route), - emqttd_trie:delete(Topic); - [_|_] -> %% Remove route only - mnesia:delete_object(Route); - [] -> ok - end. - -%% @doc Has route? --spec(has_route(binary()) -> boolean()). -has_route(Topic) when is_binary(Topic) -> - ets:member(mqtt_route, Topic). - -%% @private --spec(trans(function(), list(any())) -> ok | {error, term()}). -trans(Fun, Args) -> - case mnesia:transaction(Fun, Args) of - {atomic, _} -> ok; - {aborted, Error} -> {error, Error} - end. - -%%-------------------------------------------------------------------- -%% Local Route API -%%-------------------------------------------------------------------- - --spec(get_local_routes() -> list({binary(), node()})). -get_local_routes() -> - ets:tab2list(mqtt_local_route). - --spec(add_local_route(binary()) -> ok). -add_local_route(Topic) -> - gen_server:call(?ROUTER, {add_local_route, Topic}). - --spec(del_local_route(binary()) -> ok). -del_local_route(Topic) -> - gen_server:call(?ROUTER, {del_local_route, Topic}). - --spec(match_local(binary()) -> [mqtt_route()]). -match_local(Name) -> - case ets:info(mqtt_local_route, size) of - 0 -> []; - _ -> ets:foldl( - fun({Filter, Node}, Matched) -> - case emqttd_topic:match(Name, Filter) of - true -> [#mqtt_route{topic = {local, Filter}, node = Node} | Matched]; - false -> Matched - end - end, [], mqtt_local_route) - end. - --spec(clean_local_routes() -> ok). -clean_local_routes() -> - gen_server:call(?ROUTER, clean_local_routes). - -dump() -> - [{route, ets:tab2list(mqtt_route)}, {local_route, ets:tab2list(mqtt_local_route)}]. - -%% For unit test. -start() -> - gen_server:start({local, ?ROUTER}, ?MODULE, [], []). - -stop() -> - gen_server:call(?ROUTER, stop). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([]) -> - ekka:monitor(membership), - ets:new(mqtt_local_route, [set, named_table, protected]), - {ok, TRef} = timer:send_interval(timer:seconds(1), stats), - {ok, #state{stats_timer = TRef}}. - -handle_call({add_local_route, Topic}, _From, State) -> - %% why node()...? - ets:insert(mqtt_local_route, {Topic, node()}), - {reply, ok, State}; - -handle_call({del_local_route, Topic}, _From, State) -> - ets:delete(mqtt_local_route, Topic), - {reply, ok, State}; - -handle_call(clean_local_routes, _From, State) -> - ets:delete_all_objects(mqtt_local_route), - {reply, ok, State}; - -handle_call(stop, _From, State) -> - {stop, normal, ok, State}; - -handle_call(_Req, _From, State) -> - {reply, ignore, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info({membership, {mnesia, down, Node}}, State) -> - global:trans({?LOCK, self()}, - fun() -> - clean_routes_(Node), - update_stats_() - end), - {noreply, State, hibernate}; - -handle_info({membership, _Event}, State) -> - %% ignore - {noreply, State}; - -handle_info(stats, State) -> - update_stats_(), - {noreply, State, hibernate}; - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, #state{stats_timer = TRef}) -> - timer:cancel(TRef), - ekka:unmonitor(membership). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal Functions -%%-------------------------------------------------------------------- - -%% Clean Routes on Node -clean_routes_(Node) -> - Pattern = #mqtt_route{_ = '_', node = Node}, - Clean = fun() -> - [mnesia:delete_object(mqtt_route, R, write) || - R <- mnesia:match_object(mqtt_route, Pattern, write)] - end, - mnesia:transaction(Clean). - -update_stats_() -> - Size = mnesia:table_info(mqtt_route, size), - emqttd_stats:setstats('routes/count', 'routes/max', Size), - emqttd_stats:setstats('topics/count', 'topics/max', Size). - diff --git a/src/emqttd_serializer.erl b/src/emqttd_serializer.erl deleted file mode 100644 index 1b81a45be..000000000 --- a/src/emqttd_serializer.erl +++ /dev/null @@ -1,150 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc MQTT Packet Serializer --module(emqttd_serializer). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - -%% API --export([serialize/1]). - -%% @doc Serialise MQTT Packet --spec(serialize(mqtt_packet()) -> iolist()). -serialize(#mqtt_packet{header = Header = #mqtt_packet_header{type = Type}, - variable = Variable, - payload = Payload}) -> - serialize_header(Header, - serialize_variable(Type, Variable, - serialize_payload(Payload))). - -serialize_header(#mqtt_packet_header{type = Type, - dup = Dup, - qos = Qos, - retain = Retain}, - {VariableBin, PayloadBin}) - when ?CONNECT =< Type andalso Type =< ?DISCONNECT -> - Len = byte_size(VariableBin) + byte_size(PayloadBin), - true = (Len =< ?MAX_PACKET_SIZE), - [<>, - serialize_len(Len), VariableBin, PayloadBin]. - -serialize_variable(?CONNECT, #mqtt_packet_connect{client_id = ClientId, - proto_ver = ProtoVer, - proto_name = ProtoName, - will_retain = WillRetain, - will_qos = WillQos, - will_flag = WillFlag, - clean_sess = CleanSess, - keep_alive = KeepAlive, - will_topic = WillTopic, - will_msg = WillMsg, - username = Username, - password = Password}, undefined) -> - VariableBin = <<(byte_size(ProtoName)):16/big-unsigned-integer, - ProtoName/binary, - ProtoVer:8, - (opt(Username)):1, - (opt(Password)):1, - (opt(WillRetain)):1, - WillQos:2, - (opt(WillFlag)):1, - (opt(CleanSess)):1, - 0:1, - KeepAlive:16/big-unsigned-integer>>, - PayloadBin = serialize_utf(ClientId), - PayloadBin1 = case WillFlag of - true -> <>; - false -> PayloadBin - end, - UserPasswd = << <<(serialize_utf(B))/binary>> || B <- [Username, Password], B =/= undefined >>, - {VariableBin, <>}; - -serialize_variable(?CONNACK, #mqtt_packet_connack{ack_flags = AckFlags, - return_code = ReturnCode}, undefined) -> - {<>, <<>>}; - -serialize_variable(?SUBSCRIBE, #mqtt_packet_subscribe{packet_id = PacketId, - topic_table = Topics }, undefined) -> - {<>, serialize_topics(Topics)}; - -serialize_variable(?SUBACK, #mqtt_packet_suback{packet_id = PacketId, - qos_table = QosTable}, undefined) -> - {<>, << <> || Q <- QosTable >>}; - -serialize_variable(?UNSUBSCRIBE, #mqtt_packet_unsubscribe{packet_id = PacketId, - topics = Topics }, undefined) -> - {<>, serialize_topics(Topics)}; - -serialize_variable(?UNSUBACK, #mqtt_packet_unsuback{packet_id = PacketId}, undefined) -> - {<>, <<>>}; - -serialize_variable(?PUBLISH, #mqtt_packet_publish{topic_name = TopicName, - packet_id = PacketId }, PayloadBin) -> - TopicBin = serialize_utf(TopicName), - PacketIdBin = if - PacketId =:= undefined -> <<>>; - true -> <> - end, - {<>, PayloadBin}; - -serialize_variable(PubAck, #mqtt_packet_puback{packet_id = PacketId}, _Payload) - when PubAck =:= ?PUBACK; PubAck =:= ?PUBREC; PubAck =:= ?PUBREL; PubAck =:= ?PUBCOMP -> - {<>, <<>>}; - -serialize_variable(?PINGREQ, undefined, undefined) -> - {<<>>, <<>>}; - -serialize_variable(?PINGRESP, undefined, undefined) -> - {<<>>, <<>>}; - -serialize_variable(?DISCONNECT, undefined, undefined) -> - {<<>>, <<>>}. - -serialize_payload(undefined) -> - undefined; -serialize_payload(Bin) when is_binary(Bin) -> - Bin. - -serialize_topics([{_Topic, _Qos}|_] = Topics) -> - << <<(serialize_utf(Topic))/binary, ?RESERVED:6, Qos:2>> || {Topic, Qos} <- Topics >>; - -serialize_topics([H|_] = Topics) when is_binary(H) -> - << <<(serialize_utf(Topic))/binary>> || Topic <- Topics >>. - -serialize_utf(String) -> - StringBin = unicode:characters_to_binary(String), - Len = byte_size(StringBin), - true = (Len =< 16#ffff), - <>. - -serialize_len(N) when N =< ?LOWBITS -> - <<0:1, N:7>>; -serialize_len(N) -> - <<1:1, (N rem ?HIGHBIT):7, (serialize_len(N div ?HIGHBIT))/binary>>. - -opt(undefined) -> ?RESERVED; -opt(false) -> 0; -opt(true) -> 1; -opt(X) when is_integer(X) -> X; -opt(B) when is_binary(B) -> 1. diff --git a/src/emqttd_server.erl b/src/emqttd_server.erl deleted file mode 100644 index 8552054c8..000000000 --- a/src/emqttd_server.erl +++ /dev/null @@ -1,328 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_server). - --behaviour(gen_server2). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --include("emqttd_internal.hrl"). - --export([start_link/3]). - -%% PubSub API. --export([subscribe/1, subscribe/2, subscribe/3, publish/1, - unsubscribe/1, unsubscribe/2]). - -%% Async PubSub API. --export([async_subscribe/1, async_subscribe/2, async_subscribe/3, - async_unsubscribe/1, async_unsubscribe/2]). - -%% Management API. --export([setqos/3, subscriptions/1, subscribers/1, subscribed/2]). - -%% Debug API --export([dump/0]). - -%% gen_server Callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {pool, id, env, subids :: map(), submon :: emqttd_pmon:pmon()}). - -%% @doc Start the server --spec(start_link(atom(), pos_integer(), list()) -> {ok, pid()} | ignore | {error, term()}). -start_link(Pool, Id, Env) -> - gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id, Env], []). - -%%-------------------------------------------------------------------- -%% PubSub API -%%-------------------------------------------------------------------- - -%% @doc Subscribe to a Topic. --spec(subscribe(binary()) -> ok | {error, term()}). -subscribe(Topic) when is_binary(Topic) -> - subscribe(Topic, self()). - --spec(subscribe(binary(), emqttd:subscriber()) -> ok | {error, term()}). -subscribe(Topic, Subscriber) when is_binary(Topic) -> - subscribe(Topic, Subscriber, []). - --spec(subscribe(binary(), emqttd:subscriber(), [emqttd:suboption()]) -> - ok | {error, term()}). -subscribe(Topic, Subscriber, Options) when is_binary(Topic) -> - call(pick(Subscriber), {subscribe, Topic, with_subpid(Subscriber), Options}). - -%% @doc Subscribe to a Topic asynchronously. --spec(async_subscribe(binary()) -> ok). -async_subscribe(Topic) when is_binary(Topic) -> - async_subscribe(Topic, self()). - --spec(async_subscribe(binary(), emqttd:subscriber()) -> ok). -async_subscribe(Topic, Subscriber) when is_binary(Topic) -> - async_subscribe(Topic, Subscriber, []). - --spec(async_subscribe(binary(), emqttd:subscriber(), [emqttd:suboption()]) -> ok). -async_subscribe(Topic, Subscriber, Options) when is_binary(Topic) -> - cast(pick(Subscriber), {subscribe, Topic, with_subpid(Subscriber), Options}). - -%% @doc Publish message to Topic. --spec(publish(mqtt_message()) -> {ok, mqtt_delivery()} | ignore). -publish(Msg = #mqtt_message{from = From}) -> - trace(publish, From, Msg), - case emqttd_hooks:run('message.publish', [], Msg) of - {ok, Msg1 = #mqtt_message{topic = Topic}} -> - emqttd_pubsub:publish(Topic, Msg1); - {stop, Msg1} -> - lager:info("Stop publishing: ~s", [emqttd_message:format(Msg1)]), - ignore - end. - -%% @private -trace(publish, From, _Msg) when is_atom(From) -> - %% Dont' trace '$SYS' publish - ignore; -trace(publish, {ClientId, Username}, #mqtt_message{topic = Topic, payload = Payload}) -> - lager:debug([{client, ClientId}, {topic, Topic}], - "~s/~s PUBLISH to ~s: ~p", [ClientId, Username, Topic, Payload]); -trace(publish, From, #mqtt_message{topic = Topic, payload = Payload}) -> - lager:debug([{client, From}, {topic, Topic}], - "~s PUBLISH to ~s: ~p", [From, Topic, Payload]). - -%% @doc Unsubscribe --spec(unsubscribe(binary()) -> ok | {error, term()}). -unsubscribe(Topic) when is_binary(Topic) -> - unsubscribe(Topic, self()). - -%% @doc Unsubscribe --spec(unsubscribe(binary(), emqttd:subscriber()) -> ok | {error, term()}). -unsubscribe(Topic, Subscriber) when is_binary(Topic) -> - call(pick(Subscriber), {unsubscribe, Topic, with_subpid(Subscriber)}). - -%% @doc Async Unsubscribe --spec(async_unsubscribe(binary()) -> ok). -async_unsubscribe(Topic) when is_binary(Topic) -> - async_unsubscribe(Topic, self()). - --spec(async_unsubscribe(binary(), emqttd:subscriber()) -> ok). -async_unsubscribe(Topic, Subscriber) when is_binary(Topic) -> - cast(pick(Subscriber), {unsubscribe, Topic, with_subpid(Subscriber)}). - --spec(setqos(binary(), emqttd:subscriber(), mqtt_qos()) -> ok). -setqos(Topic, Subscriber, Qos) when is_binary(Topic) -> - call(pick(Subscriber), {setqos, Topic, with_subpid(Subscriber), Qos}). - -with_subpid(SubPid) when is_pid(SubPid) -> - SubPid; -with_subpid(SubId) when is_binary(SubId) -> - {SubId, self()}; -with_subpid({SubId, SubPid}) when is_binary(SubId), is_pid(SubPid) -> - {SubId, SubPid}. - --spec(subscriptions(emqttd:subscriber()) -> [{emqttd:subscriber(), binary(), list(emqttd:suboption())}]). -subscriptions(SubPid) when is_pid(SubPid) -> - with_subproperty(ets:lookup(mqtt_subscription, SubPid)); - -subscriptions(SubId) when is_binary(SubId) -> - with_subproperty(ets:match_object(mqtt_subscription, {{SubId, '_'}, '_'})); - -subscriptions({SubId, SubPid}) when is_binary(SubId), is_pid(SubPid) -> - with_subproperty(ets:lookup(mqtt_subscription, {SubId, SubPid})). - -with_subproperty({Subscriber, {share, _Share, Topic}}) -> - with_subproperty({Subscriber, Topic}); -with_subproperty({Subscriber, Topic}) -> - {Subscriber, Topic, ets:lookup_element(mqtt_subproperty, {Topic, Subscriber}, 2)}; -with_subproperty(Subscriptions) when is_list(Subscriptions) -> - [with_subproperty(Subscription) || Subscription <- Subscriptions]. - --spec(subscribers(binary()) -> list(emqttd:subscriber())). -subscribers(Topic) when is_binary(Topic) -> - emqttd_pubsub:subscribers(Topic). - --spec(subscribed(binary(), emqttd:subscriber()) -> boolean()). -subscribed(Topic, SubPid) when is_binary(Topic), is_pid(SubPid) -> - ets:member(mqtt_subproperty, {Topic, SubPid}); -subscribed(Topic, SubId) when is_binary(Topic), is_binary(SubId) -> - length(ets:match_object(mqtt_subproperty, {{Topic, {SubId, '_'}}, '_'}, 1)) == 1; -subscribed(Topic, {SubId, SubPid}) when is_binary(Topic), is_binary(SubId), is_pid(SubPid) -> - ets:member(mqtt_subproperty, {Topic, {SubId, SubPid}}). - -call(Server, Req) -> - gen_server2:call(Server, Req, infinity). - -cast(Server, Msg) when is_pid(Server) -> - gen_server2:cast(Server, Msg). - -pick(SubPid) when is_pid(SubPid) -> - gproc_pool:pick_worker(server, SubPid); -pick(SubId) when is_binary(SubId) -> - gproc_pool:pick_worker(server, SubId); -pick({SubId, SubPid}) when is_binary(SubId), is_pid(SubPid) -> - pick(SubId). - -dump() -> - [{Tab, ets:tab2list(Tab)} || Tab <- [mqtt_subproperty, mqtt_subscription, mqtt_subscriber]]. - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id, Env]) -> - ?GPROC_POOL(join, Pool, Id), - State = #state{pool = Pool, id = Id, env = Env, - subids = #{}, submon = emqttd_pmon:new()}, - {ok, State, hibernate, {backoff, 2000, 2000, 20000}}. - -handle_call({subscribe, Topic, Subscriber, Options}, _From, State) -> - case do_subscribe(Topic, Subscriber, Options, State) of - {ok, NewState} -> reply(ok, setstats(NewState)); - {error, Error} -> reply({error, Error}, State) - end; - -handle_call({unsubscribe, Topic, Subscriber}, _From, State) -> - case do_unsubscribe(Topic, Subscriber, State) of - {ok, NewState} -> reply(ok, setstats(NewState)); - {error, Error} -> reply({error, Error}, State) - end; - -handle_call({setqos, Topic, Subscriber, Qos}, _From, State) -> - Key = {Topic, Subscriber}, - case ets:lookup(mqtt_subproperty, Key) of - [{_, Opts}] -> - Opts1 = lists:ukeymerge(1, [{qos, Qos}], Opts), - ets:insert(mqtt_subproperty, {Key, Opts1}), - reply(ok, State); - [] -> - reply({error, {subscription_not_found, Topic}}, State) - end; - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast({subscribe, Topic, Subscriber, Options}, State) -> - case do_subscribe(Topic, Subscriber, Options, State) of - {ok, NewState} -> noreply(setstats(NewState)); - {error, _Error} -> noreply(State) - end; - -handle_cast({unsubscribe, Topic, Subscriber}, State) -> - case do_unsubscribe(Topic, Subscriber, State) of - {ok, NewState} -> noreply(setstats(NewState)); - {error, _Error} -> noreply(State) - end; - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #state{subids = SubIds}) -> - case maps:find(DownPid, SubIds) of - {ok, SubId} -> - clean_subscriber({SubId, DownPid}); - error -> - clean_subscriber(DownPid) - end, - noreply(setstats(demonitor_subscriber(DownPid, State))); - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - ?GPROC_POOL(leave, Pool, Id). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal Functions -%%-------------------------------------------------------------------- - -do_subscribe(Topic, Subscriber, Options, State) -> - case ets:lookup(mqtt_subproperty, {Topic, Subscriber}) of - [] -> - emqttd_pubsub:async_subscribe(Topic, Subscriber, Options), - Share = proplists:get_value(share, Options), - add_subscription(Share, Subscriber, Topic), - ets:insert(mqtt_subproperty, {{Topic, Subscriber}, Options}), - {ok, monitor_subscriber(Subscriber, State)}; - [_] -> - {error, {already_subscribed, Topic}} - end. - -add_subscription(undefined, Subscriber, Topic) -> - ets:insert(mqtt_subscription, {Subscriber, Topic}); -add_subscription(Share, Subscriber, Topic) -> - ets:insert(mqtt_subscription, {Subscriber, {share, Share, Topic}}). - -monitor_subscriber(SubPid, State = #state{submon = SubMon}) when is_pid(SubPid) -> - State#state{submon = SubMon:monitor(SubPid)}; -monitor_subscriber({SubId, SubPid}, State = #state{subids = SubIds, submon = SubMon}) -> - State#state{subids = maps:put(SubPid, SubId, SubIds), submon = SubMon:monitor(SubPid)}. - -do_unsubscribe(Topic, Subscriber, State) -> - case ets:lookup(mqtt_subproperty, {Topic, Subscriber}) of - [{_, Options}] -> - emqttd_pubsub:async_unsubscribe(Topic, Subscriber, Options), - Share = proplists:get_value(share, Options), - del_subscription(Share, Subscriber, Topic), - ets:delete(mqtt_subproperty, {Topic, Subscriber}), - {ok, State}; - [] -> - {error, {subscription_not_found, Topic}} - end. - -del_subscription(undefined, Subscriber, Topic) -> - ets:delete_object(mqtt_subscription, {Subscriber, Topic}); -del_subscription(Share, Subscriber, Topic) -> - ets:delete_object(mqtt_subscription, {Subscriber, {share, Share, Topic}}). - -clean_subscriber(Subscriber) -> - lists:foreach(fun({_, {share, Share, Topic}}) -> - clean_subscriber(Share, Subscriber, Topic); - ({_, Topic}) -> - clean_subscriber(undefined, Subscriber, Topic) - end, ets:lookup(mqtt_subscription, Subscriber)), - ets:delete(mqtt_subscription, Subscriber). - -clean_subscriber(Share, Subscriber, Topic) -> - case ets:lookup(mqtt_subproperty, {Topic, Subscriber}) of - [] -> - %% TODO:....??? - Options = if Share == undefined -> []; true -> [{share, Share}] end, - emqttd_pubsub:async_unsubscribe(Topic, Subscriber, Options); - [{_, Options}] -> - emqttd_pubsub:async_unsubscribe(Topic, Subscriber, Options), - ets:delete(mqtt_subproperty, {Topic, Subscriber}) - end. - -demonitor_subscriber(SubPid, State = #state{subids = SubIds, submon = SubMon}) -> - State#state{subids = maps:remove(SubPid, SubIds), submon = SubMon:demonitor(SubPid)}. - -setstats(State) -> - emqttd_stats:setstats('subscriptions/count', 'subscriptions/max', - ets:info(mqtt_subscription, size)), State. - -reply(Reply, State) -> - {reply, Reply, State, hibernate}. - -noreply(State) -> - {noreply, State, hibernate}. - diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl deleted file mode 100644 index 0d27a1cbd..000000000 --- a/src/emqttd_session.erl +++ /dev/null @@ -1,856 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% -%% @doc MQTT Session -%% -%% A stateful interaction between a Client and a Server. Some Sessions -%% last only as long as the Network Connection, others can span multiple -%% consecutive Network Connections between a Client and a Server. -%% -%% The Session state in the Server consists of: -%% -%% The existence of a Session, even if the rest of the Session state is empty. -%% -%% The Client’s subscriptions. -%% -%% QoS 1 and QoS 2 messages which have been sent to the Client, but have not -%% been completely acknowledged. -%% -%% QoS 1 and QoS 2 messages pending transmission to the Client. -%% -%% QoS 2 messages which have been received from the Client, but have not -%% been completely acknowledged. -%% -%% Optionally, QoS 0 messages pending transmission to the Client. -%% -%% If the session is currently disconnected, the time at which the Session state -%% will be deleted. -%% -%% @end -%% - --module(emqttd_session). - --behaviour(gen_server2). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --include("emqttd_internal.hrl"). - --import(emqttd_misc, [start_timer/2]). - --import(proplists, [get_value/2, get_value/3]). - -%% Session API --export([start_link/3, resume/3, destroy/2]). - -%% Management and Monitor API --export([state/1, info/1, stats/1]). - -%% PubSub API --export([subscribe/2, subscribe/3, publish/2, puback/2, pubrec/2, - pubrel/2, pubcomp/2, unsubscribe/2]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - -%% gen_server2 Message Priorities --export([prioritise_call/4, prioritise_cast/3, prioritise_info/3, - handle_pre_hibernate/1]). - --define(MQueue, emqttd_mqueue). - --record(state, - { - %% Clean Session Flag - clean_sess = false :: boolean(), - - %% Client Binding: local | remote - binding = local :: local | remote, - - %% ClientId: Identifier of Session - client_id :: binary(), - - %% Username - username :: binary() | undefined, - - %% Client Pid binding with session - client_pid :: pid(), - - %% Old Client Pid that has been kickout - old_client_pid :: pid(), - - %% Next message id of the session - next_msg_id = 1 :: mqtt_packet_id(), - - max_subscriptions :: non_neg_integer(), - - %% Client’s subscriptions. - subscriptions :: map(), - - %% Upgrade Qos? - upgrade_qos = false :: boolean(), - - %% Client <- Broker: Inflight QoS1, QoS2 messages sent to the client but unacked. - inflight :: emqttd_inflight:inflight(), - - %% Max Inflight Size - max_inflight = 32 :: non_neg_integer(), - - %% Retry interval for redelivering QoS1/2 messages - retry_interval = 20000 :: timeout(), - - %% Retry Timer - retry_timer :: reference() | undefined, - - %% All QoS1, QoS2 messages published to when client is disconnected. - %% QoS 1 and QoS 2 messages pending transmission to the Client. - %% - %% Optionally, QoS 0 messages pending transmission to the Client. - mqueue :: ?MQueue:mqueue(), - - %% Client -> Broker: Inflight QoS2 messages received from client and waiting for pubrel. - awaiting_rel :: map(), - - %% Max Packets that Awaiting PUBREL - max_awaiting_rel = 100 :: non_neg_integer(), - - %% Awaiting PUBREL timeout - await_rel_timeout = 20000 :: timeout(), - - %% Awaiting PUBREL timer - await_rel_timer :: reference() | undefined, - - %% Session Expiry Interval - expiry_interval = 7200000 :: timeout(), - - %% Expired Timer - expiry_timer :: reference() | undefined, - - %% Enable Stats - enable_stats :: boolean(), - - %% Force GC Count - force_gc_count :: undefined | integer(), - - %% Ignore loop deliver? - ignore_loop_deliver = false :: boolean(), - - created_at :: erlang:timestamp() - }). - --define(TIMEOUT, 60000). - --define(INFO_KEYS, [clean_sess, client_id, username, client_pid, binding, created_at]). - --define(STATE_KEYS, [clean_sess, client_id, username, binding, client_pid, old_client_pid, - next_msg_id, max_subscriptions, subscriptions, upgrade_qos, inflight, - max_inflight, retry_interval, mqueue, awaiting_rel, max_awaiting_rel, - await_rel_timeout, expiry_interval, enable_stats, force_gc_count, - created_at]). - --define(LOG(Level, Format, Args, State), - lager:Level([{client, State#state.client_id}], - "Session(~s): " ++ Format, [State#state.client_id | Args])). - -%% @doc Start a Session --spec(start_link(boolean(), {mqtt_client_id(), mqtt_username()}, pid()) -> {ok, pid()} | {error, term()}). -start_link(CleanSess, {ClientId, Username}, ClientPid) -> - gen_server2:start_link(?MODULE, [CleanSess, {ClientId, Username}, ClientPid], []). - -%%-------------------------------------------------------------------- -%% PubSub API -%%-------------------------------------------------------------------- - -%% @doc Subscribe topics --spec(subscribe(pid(), [{binary(), [emqttd_topic:option()]}]) -> ok). -subscribe(Session, TopicTable) -> %%TODO: the ack function??... - gen_server2:cast(Session, {subscribe, self(), TopicTable, fun(_) -> ok end}). - --spec(subscribe(pid(), mqtt_packet_id(), [{binary(), [emqttd_topic:option()]}]) -> ok). -subscribe(Session, PacketId, TopicTable) -> %%TODO: the ack function??... - From = self(), - AckFun = fun(GrantedQos) -> From ! {suback, PacketId, GrantedQos} end, - gen_server2:cast(Session, {subscribe, From, TopicTable, AckFun}). - -%% @doc Publish Message --spec(publish(pid(), mqtt_message()) -> ok | {error, term()}). -publish(_Session, Msg = #mqtt_message{qos = ?QOS_0}) -> - %% Publish QoS0 Directly - emqttd_server:publish(Msg), ok; - -publish(_Session, Msg = #mqtt_message{qos = ?QOS_1}) -> - %% Publish QoS1 message directly for client will PubAck automatically - emqttd_server:publish(Msg), ok; - -publish(Session, Msg = #mqtt_message{qos = ?QOS_2}) -> - %% Publish QoS2 to Session - gen_server2:call(Session, {publish, Msg}, ?TIMEOUT). - -%% @doc PubAck Message --spec(puback(pid(), mqtt_packet_id()) -> ok). -puback(Session, PacketId) -> - gen_server2:cast(Session, {puback, PacketId}). - --spec(pubrec(pid(), mqtt_packet_id()) -> ok). -pubrec(Session, PacketId) -> - gen_server2:cast(Session, {pubrec, PacketId}). - --spec(pubrel(pid(), mqtt_packet_id()) -> ok). -pubrel(Session, PacketId) -> - gen_server2:cast(Session, {pubrel, PacketId}). - --spec(pubcomp(pid(), mqtt_packet_id()) -> ok). -pubcomp(Session, PacketId) -> - gen_server2:cast(Session, {pubcomp, PacketId}). - -%% @doc Unsubscribe the topics --spec(unsubscribe(pid(), [{binary(), [emqttd_topic:option()]}]) -> ok). -unsubscribe(Session, TopicTable) -> - gen_server2:cast(Session, {unsubscribe, self(), TopicTable}). - -%% @doc Resume the session --spec(resume(pid(), mqtt_client_id(), pid()) -> ok). -resume(Session, ClientId, ClientPid) -> - gen_server2:cast(Session, {resume, ClientId, ClientPid}). - -%% @doc Get session state -state(Session) when is_pid(Session) -> - gen_server2:call(Session, state). - -%% @doc Get session info --spec(info(pid() | #state{}) -> list(tuple())). -info(Session) when is_pid(Session) -> - gen_server2:call(Session, info); - -info(State) when is_record(State, state) -> - ?record_to_proplist(state, State, ?INFO_KEYS). - --spec(stats(pid() | #state{}) -> list({atom(), non_neg_integer()})). -stats(Session) when is_pid(Session) -> - gen_server2:call(Session, stats); - -stats(#state{max_subscriptions = MaxSubscriptions, - subscriptions = Subscriptions, - inflight = Inflight, - max_inflight = MaxInflight, - mqueue = MQueue, - max_awaiting_rel = MaxAwaitingRel, - awaiting_rel = AwaitingRel}) -> - lists:append(emqttd_misc:proc_stats(), - [{max_subscriptions, MaxSubscriptions}, - {subscriptions, maps:size(Subscriptions)}, - {max_inflight, MaxInflight}, - {inflight_len, Inflight:size()}, - {max_mqueue, ?MQueue:max_len(MQueue)}, - {mqueue_len, ?MQueue:len(MQueue)}, - {mqueue_dropped, ?MQueue:dropped(MQueue)}, - {max_awaiting_rel, MaxAwaitingRel}, - {awaiting_rel_len, maps:size(AwaitingRel)}, - {deliver_msg, get(deliver_msg)}, - {enqueue_msg, get(enqueue_msg)}]). - -%% @doc Destroy the session --spec(destroy(pid(), mqtt_client_id()) -> ok). -destroy(Session, ClientId) -> - gen_server2:cast(Session, {destroy, ClientId}). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([CleanSess, {ClientId, Username}, ClientPid]) -> - process_flag(trap_exit, true), - true = link(ClientPid), - init_stats([deliver_msg, enqueue_msg]), - {ok, Env} = emqttd:env(session), - {ok, QEnv} = emqttd:env(mqueue), - MaxInflight = get_value(max_inflight, Env, 0), - EnableStats = get_value(enable_stats, Env, false), - ForceGcCount = emqttd_gc:conn_max_gc_count(), - IgnoreLoopDeliver = get_value(ignore_loop_deliver, Env, false), - MQueue = ?MQueue:new(ClientId, QEnv, emqttd_alarm:alarm_fun()), - State = #state{clean_sess = CleanSess, - binding = binding(ClientPid), - client_id = ClientId, - client_pid = ClientPid, - username = Username, - subscriptions = #{}, - max_subscriptions = get_value(max_subscriptions, Env, 0), - upgrade_qos = get_value(upgrade_qos, Env, false), - max_inflight = MaxInflight, - inflight = emqttd_inflight:new(MaxInflight), - mqueue = MQueue, - retry_interval = get_value(retry_interval, Env), - awaiting_rel = #{}, - await_rel_timeout = get_value(await_rel_timeout, Env), - max_awaiting_rel = get_value(max_awaiting_rel, Env), - expiry_interval = get_value(expiry_interval, Env), - enable_stats = EnableStats, - force_gc_count = ForceGcCount, - created_at = os:timestamp(), - ignore_loop_deliver = IgnoreLoopDeliver}, - emqttd_sm:register_session(ClientId, CleanSess, info(State)), - emqttd_hooks:run('session.created', [ClientId, Username]), - {ok, emit_stats(State), hibernate, {backoff, 1000, 1000, 10000}}. - -init_stats(Keys) -> - lists:foreach(fun(K) -> put(K, 0) end, Keys). - -binding(ClientPid) -> - case node(ClientPid) =:= node() of true -> local; false -> remote end. - -prioritise_call(Msg, _From, _Len, _State) -> - case Msg of info -> 10; stats -> 10; state -> 10; _ -> 5 end. - -prioritise_cast(Msg, _Len, _State) -> - case Msg of - {destroy, _} -> 10; - {resume, _, _} -> 9; - {pubrel, _} -> 8; - {pubcomp, _} -> 8; - {pubrec, _} -> 8; - {puback, _} -> 7; - {unsubscribe, _, _} -> 6; - {subscribe, _, _} -> 5; - _ -> 0 - end. - -prioritise_info(Msg, _Len, _State) -> - case Msg of - {'EXIT', _, _} -> 10; - {timeout, _, _} -> 5; - {dispatch, _, _} -> 1; - _ -> 0 - end. - -handle_pre_hibernate(State) -> - {hibernate, emqttd_gc:reset_conn_gc_count(#state.force_gc_count, emit_stats(State))}. - -handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PacketId}}, _From, - State = #state{awaiting_rel = AwaitingRel, - await_rel_timer = Timer, - await_rel_timeout = Timeout}) -> - case is_awaiting_full(State) of - false -> - State1 = case Timer == undefined of - true -> State#state{await_rel_timer = start_timer(Timeout, check_awaiting_rel)}; - false -> State - end, - reply(ok, State1#state{awaiting_rel = maps:put(PacketId, Msg, AwaitingRel)}); - true -> - ?LOG(warning, "Dropped Qos2 Message for too many awaiting_rel: ~p", [Msg], State), - emqttd_metrics:inc('messages/qos2/dropped'), - reply({error, dropped}, State) - end; - -handle_call(info, _From, State) -> - reply(info(State), State); - -handle_call(stats, _From, State) -> - reply(stats(State), State); - -handle_call(state, _From, State) -> - reply(?record_to_proplist(state, State, ?STATE_KEYS), State); - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast({subscribe, _From, TopicTable, AckFun}, - State = #state{client_id = ClientId, - username = Username, - subscriptions = Subscriptions}) -> - ?LOG(debug, "Subscribe ~p", [TopicTable], State), - {GrantedQos, Subscriptions1} = - lists:foldl(fun({Topic, Opts}, {QosAcc, SubMap}) -> - NewQos = proplists:get_value(qos, Opts), - SubMap1 = - case maps:find(Topic, SubMap) of - {ok, NewQos} -> - emqttd_hooks:run('session.subscribed', [ClientId, Username], {Topic, Opts}), - ?LOG(warning, "Duplicated subscribe: ~s, qos = ~w", [Topic, NewQos], State), - SubMap; - {ok, OldQos} -> - emqttd:setqos(Topic, ClientId, NewQos), - emqttd_hooks:run('session.subscribed', [ClientId, Username], {Topic, Opts}), - ?LOG(warning, "Duplicated subscribe ~s, old_qos=~w, new_qos=~w", - [Topic, OldQos, NewQos], State), - maps:put(Topic, NewQos, SubMap); - error -> - emqttd:subscribe(Topic, ClientId, Opts), - emqttd_hooks:run('session.subscribed', [ClientId, Username], {Topic, Opts}), - maps:put(Topic, NewQos, SubMap) - end, - {[NewQos|QosAcc], SubMap1} - end, {[], Subscriptions}, TopicTable), - AckFun(lists:reverse(GrantedQos)), - hibernate(emit_stats(State#state{subscriptions = Subscriptions1})); - -handle_cast({unsubscribe, _From, TopicTable}, - State = #state{client_id = ClientId, - username = Username, - subscriptions = Subscriptions}) -> - ?LOG(debug, "Unsubscribe ~p", [TopicTable], State), - Subscriptions1 = - lists:foldl(fun({Topic, Opts}, SubMap) -> - case maps:find(Topic, SubMap) of - {ok, _Qos} -> - emqttd:unsubscribe(Topic, ClientId), - emqttd_hooks:run('session.unsubscribed', [ClientId, Username], {Topic, Opts}), - maps:remove(Topic, SubMap); - error -> - SubMap - end - end, Subscriptions, TopicTable), - hibernate(emit_stats(State#state{subscriptions = Subscriptions1})); - -%% PUBACK: -handle_cast({puback, PacketId}, State = #state{inflight = Inflight}) -> - {noreply, - case Inflight:contain(PacketId) of - true -> - dequeue(acked(puback, PacketId, State)); - false -> - ?LOG(warning, "PUBACK ~p missed inflight: ~p", - [PacketId, Inflight:window()], State), - emqttd_metrics:inc('packets/puback/missed'), - State - end, hibernate}; - -%% PUBREC: -handle_cast({pubrec, PacketId}, State = #state{inflight = Inflight}) -> - {noreply, - case Inflight:contain(PacketId) of - true -> - acked(pubrec, PacketId, State); - false -> - ?LOG(warning, "PUBREC ~p missed inflight: ~p", - [PacketId, Inflight:window()], State), - emqttd_metrics:inc('packets/pubrec/missed'), - State - end, hibernate}; - -%% PUBREL: -handle_cast({pubrel, PacketId}, State = #state{awaiting_rel = AwaitingRel}) -> - {noreply, - case maps:take(PacketId, AwaitingRel) of - {Msg, AwaitingRel1} -> - %% Implement Qos2 by method A [MQTT 4.33] - %% Dispatch to subscriber when received PUBREL - spawn(emqttd_server, publish, [Msg]), %%:) - gc(State#state{awaiting_rel = AwaitingRel1}); - error -> - ?LOG(warning, "Cannot find PUBREL: ~p", [PacketId], State), - emqttd_metrics:inc('packets/pubrel/missed'), - State - end, hibernate}; - -%% PUBCOMP: -handle_cast({pubcomp, PacketId}, State = #state{inflight = Inflight}) -> - {noreply, - case Inflight:contain(PacketId) of - true -> - dequeue(acked(pubcomp, PacketId, State)); - false -> - ?LOG(warning, "The PUBCOMP ~p is not inflight: ~p", - [PacketId, Inflight:window()], State), - emqttd_metrics:inc('packets/pubcomp/missed'), - State - end, hibernate}; - -%% RESUME: -handle_cast({resume, ClientId, ClientPid}, - State = #state{client_id = ClientId, - client_pid = OldClientPid, - clean_sess = CleanSess, - retry_timer = RetryTimer, - await_rel_timer = AwaitTimer, - expiry_timer = ExpireTimer}) -> - - ?LOG(debug, "Resumed by ~p", [ClientPid], State), - - %% Cancel Timers - lists:foreach(fun emqttd_misc:cancel_timer/1, - [RetryTimer, AwaitTimer, ExpireTimer]), - - case kick(ClientId, OldClientPid, ClientPid) of - ok -> ?LOG(warning, "~p kickout ~p", [ClientPid, OldClientPid], State); - ignore -> ok - end, - - true = link(ClientPid), - - State1 = State#state{client_pid = ClientPid, - binding = binding(ClientPid), - old_client_pid = OldClientPid, - clean_sess = false, - retry_timer = undefined, - awaiting_rel = #{}, - await_rel_timer = undefined, - expiry_timer = undefined}, - - %% Clean Session: true -> false? - if - CleanSess =:= true -> - ?LOG(info, "CleanSess changed to false.", [], State1), - emqttd_sm:register_session(ClientId, false, info(State1)); - CleanSess =:= false -> - ok - end, - - %% Replay delivery and Dequeue pending messages - hibernate(emit_stats(dequeue(retry_delivery(true, State1)))); - -handle_cast({destroy, ClientId}, - State = #state{client_id = ClientId, client_pid = undefined}) -> - ?LOG(warning, "Destroyed", [], State), - shutdown(destroy, State); - -handle_cast({destroy, ClientId}, - State = #state{client_id = ClientId, client_pid = OldClientPid}) -> - ?LOG(warning, "kickout ~p", [OldClientPid], State), - shutdown(conflict, State); - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -%% Ignore Messages delivered by self -handle_info({dispatch, _Topic, #mqtt_message{from = {ClientId, _}}}, - State = #state{client_id = ClientId, ignore_loop_deliver = true}) -> - hibernate(State); - -%% Dispatch Message -handle_info({dispatch, Topic, Msg}, State) when is_record(Msg, mqtt_message) -> - hibernate(gc(dispatch(tune_qos(Topic, reset_dup(Msg), State), State))); - -%% Do nothing if the client has been disconnected. -handle_info({timeout, _Timer, retry_delivery}, State = #state{client_pid = undefined}) -> - hibernate(emit_stats(State#state{retry_timer = undefined})); - -handle_info({timeout, _Timer, retry_delivery}, State) -> - hibernate(emit_stats(retry_delivery(false, State#state{retry_timer = undefined}))); - -handle_info({timeout, _Timer, check_awaiting_rel}, State) -> - hibernate(expire_awaiting_rel(emit_stats(State#state{await_rel_timer = undefined}))); - -handle_info({timeout, _Timer, expired}, State) -> - ?LOG(info, "Expired, shutdown now.", [], State), - shutdown(expired, State); - -handle_info({'EXIT', ClientPid, _Reason}, - State = #state{clean_sess = true, client_pid = ClientPid}) -> - {stop, normal, State}; - -handle_info({'EXIT', ClientPid, Reason}, - State = #state{clean_sess = false, - client_pid = ClientPid, - expiry_interval = Interval}) -> - ?LOG(info, "Client ~p EXIT for ~p", [ClientPid, Reason], State), - ExpireTimer = start_timer(Interval, expired), - State1 = State#state{client_pid = undefined, expiry_timer = ExpireTimer}, - hibernate(emit_stats(State1)); - -handle_info({'EXIT', Pid, _Reason}, State = #state{old_client_pid = Pid}) -> - %%ignore - hibernate(State); - -handle_info({'EXIT', Pid, Reason}, State = #state{client_pid = ClientPid}) -> - - ?LOG(error, "Unexpected EXIT: client_pid=~p, exit_pid=~p, reason=~p", - [ClientPid, Pid, Reason], State), - hibernate(State); - -handle_info(Info, Session) -> - ?UNEXPECTED_INFO(Info, Session). - -terminate(Reason, #state{client_id = ClientId, username = Username}) -> - %% Move to emqttd_sm to avoid race condition - %%emqttd_stats:del_session_stats(ClientId), - emqttd_hooks:run('session.terminated', [ClientId, Username, Reason]), - emqttd_sm:unregister_session(ClientId). - -code_change(_OldVsn, Session, _Extra) -> - {ok, Session}. - -%%-------------------------------------------------------------------- -%% Kickout old client -%%-------------------------------------------------------------------- -kick(_ClientId, undefined, _Pid) -> - ignore; -kick(_ClientId, Pid, Pid) -> - ignore; -kick(ClientId, OldPid, Pid) -> - unlink(OldPid), - OldPid ! {shutdown, conflict, {ClientId, Pid}}, - %% Clean noproc - receive {'EXIT', OldPid, _} -> ok after 0 -> ok end. - -%%-------------------------------------------------------------------- -%% Replay or Retry Delivery -%%-------------------------------------------------------------------- - -%% Redeliver at once if Force is true - -retry_delivery(Force, State = #state{inflight = Inflight}) -> - case Inflight:is_empty() of - true -> State; - false -> Msgs = lists:sort(sortfun(inflight), Inflight:values()), - retry_delivery(Force, Msgs, os:timestamp(), State) - end. - -retry_delivery(_Force, [], _Now, State = #state{retry_interval = Interval}) -> - State#state{retry_timer = start_timer(Interval, retry_delivery)}; - -retry_delivery(Force, [{Type, Msg, Ts} | Msgs], Now, - State = #state{inflight = Inflight, - retry_interval = Interval}) -> - Diff = timer:now_diff(Now, Ts) div 1000, %% micro -> ms - if - Force orelse (Diff >= Interval) -> - case {Type, Msg} of - {publish, Msg = #mqtt_message{pktid = PacketId}} -> - redeliver(Msg, State), - Inflight1 = Inflight:update(PacketId, {publish, Msg, Now}), - retry_delivery(Force, Msgs, Now, State#state{inflight = Inflight1}); - {pubrel, PacketId} -> - redeliver({pubrel, PacketId}, State), - Inflight1 = Inflight:update(PacketId, {pubrel, PacketId, Now}), - retry_delivery(Force, Msgs, Now, State#state{inflight = Inflight1}) - end; - true -> - State#state{retry_timer = start_timer(Interval - Diff, retry_delivery)} - end. - -%%-------------------------------------------------------------------- -%% Expire Awaiting Rel -%%-------------------------------------------------------------------- - -expire_awaiting_rel(State = #state{awaiting_rel = AwaitingRel}) -> - case maps:size(AwaitingRel) of - 0 -> State; - _ -> Msgs = lists:sort(sortfun(awaiting_rel), maps:to_list(AwaitingRel)), - expire_awaiting_rel(Msgs, os:timestamp(), State) - end. - -expire_awaiting_rel([], _Now, State) -> - State#state{await_rel_timer = undefined}; - -expire_awaiting_rel([{PacketId, Msg = #mqtt_message{timestamp = TS}} | Msgs], - Now, State = #state{awaiting_rel = AwaitingRel, - await_rel_timeout = Timeout}) -> - case (timer:now_diff(Now, TS) div 1000) of - Diff when Diff >= Timeout -> - ?LOG(warning, "Dropped Qos2 Message for await_rel_timeout: ~p", [Msg], State), - emqttd_metrics:inc('messages/qos2/dropped'), - expire_awaiting_rel(Msgs, Now, State#state{awaiting_rel = maps:remove(PacketId, AwaitingRel)}); - Diff -> - State#state{await_rel_timer = start_timer(Timeout - Diff, check_awaiting_rel)} - end. - -%%-------------------------------------------------------------------- -%% Sort Inflight, AwaitingRel -%%-------------------------------------------------------------------- - -sortfun(inflight) -> - fun({_, _, Ts1}, {_, _, Ts2}) -> Ts1 < Ts2 end; - -sortfun(awaiting_rel) -> - fun({_, #mqtt_message{timestamp = Ts1}}, - {_, #mqtt_message{timestamp = Ts2}}) -> - Ts1 < Ts2 - end. - -%%-------------------------------------------------------------------- -%% Check awaiting rel -%%-------------------------------------------------------------------- - -is_awaiting_full(#state{max_awaiting_rel = 0}) -> - false; -is_awaiting_full(#state{awaiting_rel = AwaitingRel, max_awaiting_rel = MaxLen}) -> - maps:size(AwaitingRel) >= MaxLen. - -%%-------------------------------------------------------------------- -%% Dispatch Messages -%%-------------------------------------------------------------------- - -%% Enqueue message if the client has been disconnected -dispatch(Msg, State = #state{client_pid = undefined}) -> - enqueue_msg(Msg, State); - -%% Deliver qos0 message directly to client -dispatch(Msg = #mqtt_message{qos = ?QOS0}, State) -> - deliver(Msg, State), State; - -dispatch(Msg = #mqtt_message{qos = QoS}, - State = #state{next_msg_id = MsgId, inflight = Inflight}) - when QoS =:= ?QOS1 orelse QoS =:= ?QOS2 -> - case Inflight:is_full() of - true -> - enqueue_msg(Msg, State); - false -> - Msg1 = Msg#mqtt_message{pktid = MsgId}, - deliver(Msg1, State), - await(Msg1, next_msg_id(State)) - end. - -enqueue_msg(Msg, State = #state{mqueue = Q}) -> - inc_stats(enqueue_msg), - State#state{mqueue = ?MQueue:in(Msg, Q)}. - -%%-------------------------------------------------------------------- -%% Deliver -%%-------------------------------------------------------------------- - -redeliver(Msg = #mqtt_message{qos = QoS}, State) -> - deliver(Msg#mqtt_message{dup = if QoS =:= ?QOS0 -> false; true -> true end}, State); - -redeliver({pubrel, PacketId}, #state{client_pid = Pid}) -> - Pid ! {redeliver, {?PUBREL, PacketId}}. - -deliver(Msg, #state{client_pid = Pid}) -> - inc_stats(deliver_msg), - Pid ! {deliver, Msg}. - -%%-------------------------------------------------------------------- -%% Awaiting ACK for QoS1/QoS2 Messages -%%-------------------------------------------------------------------- - -await(Msg = #mqtt_message{pktid = PacketId}, - State = #state{inflight = Inflight, - retry_timer = RetryTimer, - retry_interval = Interval}) -> - %% Start retry timer if the Inflight is still empty - State1 = ?IF(RetryTimer == undefined, State#state{retry_timer = start_timer(Interval, retry_delivery)}, State), - State1#state{inflight = Inflight:insert(PacketId, {publish, Msg, os:timestamp()})}. - -acked(puback, PacketId, State = #state{client_id = ClientId, - username = Username, - inflight = Inflight}) -> - case Inflight:lookup(PacketId) of - {publish, Msg, _Ts} -> - emqttd_hooks:run('message.acked', [ClientId, Username], Msg), - State#state{inflight = Inflight:delete(PacketId)}; - _ -> - ?LOG(warning, "Duplicated PUBACK Packet: ~p", [PacketId], State), - State - end; - -acked(pubrec, PacketId, State = #state{client_id = ClientId, - username = Username, - inflight = Inflight}) -> - case Inflight:lookup(PacketId) of - {publish, Msg, _Ts} -> - emqttd_hooks:run('message.acked', [ClientId, Username], Msg), - State#state{inflight = Inflight:update(PacketId, {pubrel, PacketId, os:timestamp()})}; - {pubrel, PacketId, _Ts} -> - ?LOG(warning, "Duplicated PUBREC Packet: ~p", [PacketId], State), - State - end; - -acked(pubcomp, PacketId, State = #state{inflight = Inflight}) -> - State#state{inflight = Inflight:delete(PacketId)}. - -%%-------------------------------------------------------------------- -%% Dequeue -%%-------------------------------------------------------------------- - -%% Do nothing if client is disconnected -dequeue(State = #state{client_pid = undefined}) -> - State; - -dequeue(State = #state{inflight = Inflight}) -> - case Inflight:is_full() of - true -> State; - false -> dequeue2(State) - end. - -dequeue2(State = #state{mqueue = Q}) -> - case ?MQueue:out(Q) of - {empty, _Q} -> - State; - {{value, Msg}, Q1} -> - %% Dequeue more - dequeue(dispatch(Msg, State#state{mqueue = Q1})) - end. - -%%-------------------------------------------------------------------- -%% Tune QoS -%%-------------------------------------------------------------------- - -tune_qos(Topic, Msg = #mqtt_message{qos = PubQoS}, - #state{subscriptions = SubMap, upgrade_qos = UpgradeQoS}) -> - case maps:find(Topic, SubMap) of - {ok, SubQoS} when UpgradeQoS andalso (SubQoS > PubQoS) -> - Msg#mqtt_message{qos = SubQoS}; - {ok, SubQoS} when (not UpgradeQoS) andalso (SubQoS < PubQoS) -> - Msg#mqtt_message{qos = SubQoS}; - {ok, _} -> - Msg; - error -> - Msg - end. - -%%-------------------------------------------------------------------- -%% Reset Dup -%%-------------------------------------------------------------------- - -reset_dup(Msg = #mqtt_message{dup = true}) -> - Msg#mqtt_message{dup = false}; -reset_dup(Msg) -> Msg. - -%%-------------------------------------------------------------------- -%% Next Msg Id -%%-------------------------------------------------------------------- - -next_msg_id(State = #state{next_msg_id = 16#FFFF}) -> - State#state{next_msg_id = 1}; - -next_msg_id(State = #state{next_msg_id = Id}) -> - State#state{next_msg_id = Id + 1}. - -%%-------------------------------------------------------------------- -%% Emit session stats -%%-------------------------------------------------------------------- - -emit_stats(State = #state{enable_stats = false}) -> - State; -emit_stats(State = #state{client_id = ClientId}) -> - emqttd_stats:set_session_stats(ClientId, stats(State)), - State. - -inc_stats(Key) -> put(Key, get(Key) + 1). - -%%-------------------------------------------------------------------- -%% Helper functions -%%-------------------------------------------------------------------- - -reply(Reply, State) -> - {reply, Reply, State, hibernate}. - -hibernate(State) -> - {noreply, State, hibernate}. - -shutdown(Reason, State) -> - {stop, {shutdown, Reason}, State}. - -gc(State) -> - emqttd_gc:maybe_force_gc(#state.force_gc_count, State). diff --git a/src/emqttd_session_sup.erl b/src/emqttd_session_sup.erl deleted file mode 100644 index 506383834..000000000 --- a/src/emqttd_session_sup.erl +++ /dev/null @@ -1,45 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Session Supervisor. --module(emqttd_session_sup). - --author("Feng Lee "). - --behavior(supervisor). - --export([start_link/0, start_session/3]). - --export([init/1]). - -%% @doc Start session supervisor --spec(start_link() -> {ok, pid()}). -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -%% @doc Start a session --spec(start_session(boolean(), {binary(), binary() | undefined} , pid()) -> {ok, pid()}). -start_session(CleanSess, {ClientId, Username}, ClientPid) -> - supervisor:start_child(?MODULE, [CleanSess, {ClientId, Username}, ClientPid]). - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - -init([]) -> - {ok, {{simple_one_for_one, 0, 1}, - [{session, {emqttd_session, start_link, []}, - temporary, 5000, worker, [emqttd_session]}]}}. diff --git a/src/emqttd_sm.erl b/src/emqttd_sm.erl deleted file mode 100644 index 5d9b900c3..000000000 --- a/src/emqttd_sm.erl +++ /dev/null @@ -1,309 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_sm). - --author("Feng Lee "). - --behaviour(gen_server2). - --include("emqttd.hrl"). - --include("emqttd_internal.hrl"). - -%% Mnesia Callbacks --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - -%% API Function Exports --export([start_link/2]). - --export([start_session/2, lookup_session/1, register_session/3, - unregister_session/1, unregister_session/2]). - --export([dispatch/3]). - --export([local_sessions/0]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - -%% gen_server2 priorities --export([prioritise_call/4, prioritise_cast/3, prioritise_info/3]). - --record(state, {pool, id, monitors}). - --define(POOL, ?MODULE). - --define(TIMEOUT, 120000). - --define(LOG(Level, Format, Args, Session), - lager:Level("SM(~s): " ++ Format, [Session#mqtt_session.client_id | Args])). - -%%-------------------------------------------------------------------- -%% Mnesia callbacks -%%-------------------------------------------------------------------- - -mnesia(boot) -> - %% Global Session Table - ok = ekka_mnesia:create_table(mqtt_session, [ - {type, set}, - {ram_copies, [node()]}, - {record_name, mqtt_session}, - {attributes, record_info(fields, mqtt_session)}]); - -mnesia(copy) -> - ok = ekka_mnesia:copy_table(mqtt_session). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -%% @doc Start a session manager --spec(start_link(atom(), pos_integer()) -> {ok, pid()} | ignore | {error, term()}). -start_link(Pool, Id) -> - gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id], []). - -%% @doc Start a session --spec(start_session(boolean(), {binary(), binary() | undefined}) -> {ok, pid(), boolean()} | {error, term()}). -start_session(CleanSess, {ClientId, Username}) -> - SM = gproc_pool:pick_worker(?POOL, ClientId), - call(SM, {start_session, CleanSess, {ClientId, Username}, self()}). - -%% @doc Lookup a Session --spec(lookup_session(binary()) -> mqtt_session() | undefined). -lookup_session(ClientId) -> - case mnesia:dirty_read(mqtt_session, ClientId) of - [Session] -> Session; - [] -> undefined - end. - -%% @doc Register a session with info. --spec(register_session(binary(), boolean(), [tuple()]) -> true). -register_session(ClientId, CleanSess, Properties) -> - ets:insert(mqtt_local_session, {ClientId, self(), CleanSess, Properties}). - -%% @doc Unregister a session. --spec(unregister_session(binary()) -> boolean()). -unregister_session(ClientId) -> - unregister_session(ClientId, self()). - -unregister_session(ClientId, Pid) -> - case ets:lookup(mqtt_local_session, ClientId) of - [LocalSess = {_, Pid, _, _}] -> - emqttd_stats:del_session_stats(ClientId), - ets:delete_object(mqtt_local_session, LocalSess); - _ -> - false - end. - -dispatch(ClientId, Topic, Msg) -> - try ets:lookup_element(mqtt_local_session, ClientId, 2) of - Pid -> Pid ! {dispatch, Topic, Msg} - catch - error:badarg -> ok %%FIXME Later. - end. - -call(SM, Req) -> - gen_server2:call(SM, Req, ?TIMEOUT). %%infinity). - -%% @doc for debug. -local_sessions() -> - ets:tab2list(mqtt_local_session). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id]) -> - ?GPROC_POOL(join, Pool, Id), - {ok, #state{pool = Pool, id = Id, monitors = dict:new()}}. - -prioritise_call(_Msg, _From, _Len, _State) -> - 1. - -prioritise_cast(_Msg, _Len, _State) -> - 0. - -prioritise_info(_Msg, _Len, _State) -> - 2. - -%% Persistent Session -handle_call({start_session, false, {ClientId, Username}, ClientPid}, _From, State) -> - case lookup_session(ClientId) of - undefined -> - %% Create session locally - create_session({false, {ClientId, Username}, ClientPid}, State); - Session -> - case resume_session(Session, ClientPid) of - {ok, SessPid} -> - {reply, {ok, SessPid, true}, State}; - {error, Erorr} -> - {reply, {error, Erorr}, State} - end - end; - -%% Transient Session -handle_call({start_session, true, {ClientId, Username}, ClientPid}, _From, State) -> - Client = {true, {ClientId, Username}, ClientPid}, - case lookup_session(ClientId) of - undefined -> - create_session(Client, State); - Session -> - case destroy_session(Session) of - ok -> - create_session(Client, State); - {error, Error} -> - {reply, {error, Error}, State} - end - end; - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info({'DOWN', MRef, process, DownPid, _Reason}, State) -> - case dict:find(MRef, State#state.monitors) of - {ok, ClientId} -> - NewState = - case mnesia:dirty_read({mqtt_session, ClientId}) of - [] -> State; - [Sess = #mqtt_session{sess_pid = DownPid}] -> - mnesia:dirty_delete_object(Sess), - erase_monitor(MRef, State); - [_Sess] -> - State - end, - {noreply, NewState, hibernate}; - error -> - lager:error("MRef of session ~p not found", [DownPid]), - {noreply, State} - end; - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - ?GPROC_POOL(leave, Pool, Id). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -%% Create Session Locally -create_session({CleanSess, {ClientId, Username}, ClientPid}, State) -> - case create_session(CleanSess, {ClientId, Username}, ClientPid) of - {ok, SessPid} -> - {reply, {ok, SessPid, false}, monitor_session(ClientId, SessPid, State)}; - {error, Error} -> - {reply, {error, Error}, State} - end. - -create_session(CleanSess, {ClientId, Username}, ClientPid) -> - case emqttd_session_sup:start_session(CleanSess, {ClientId, Username}, ClientPid) of - {ok, SessPid} -> - Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid, clean_sess = CleanSess}, - case insert_session(Session) of - {aborted, {conflict, ConflictPid}} -> - %% Conflict with othe node? - lager:error("SM(~s): Conflict with ~p", [ClientId, ConflictPid]), - {error, mnesia_conflict}; - {atomic, ok} -> - {ok, SessPid} - end; - {error, Error} -> - {error, Error} - end. - -insert_session(Session = #mqtt_session{client_id = ClientId}) -> - mnesia:transaction( - fun() -> - case mnesia:wread({mqtt_session, ClientId}) of - [] -> - mnesia:write(mqtt_session, Session, write); - [#mqtt_session{sess_pid = SessPid}] -> - mnesia:abort({conflict, SessPid}) - end - end). - -%% Local node -resume_session(Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid}, ClientPid) - when node(SessPid) =:= node() -> - - case is_process_alive(SessPid) of - true -> - emqttd_session:resume(SessPid, ClientId, ClientPid), - {ok, SessPid}; - false -> - ?LOG(error, "Cannot resume ~p which seems already dead!", [SessPid], Session), - remove_session(Session), - {error, session_died} - end; - -%% Remote node -resume_session(Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid}, ClientPid) -> - Node = node(SessPid), - case rpc:call(Node, emqttd_session, resume, [SessPid, ClientId, ClientPid]) of - ok -> - {ok, SessPid}; - {badrpc, nodedown} -> - ?LOG(error, "Session died for node '~s' down", [Node], Session), - remove_session(Session), - {error, session_nodedown}; - {badrpc, Reason} -> - ?LOG(error, "Failed to resume from node ~s for ~p", [Node, Reason], Session), - {error, Reason} - end. - -%% Local node -destroy_session(Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid}) - when node(SessPid) =:= node() -> - emqttd_session:destroy(SessPid, ClientId), - remove_session(Session); - -%% Remote node -destroy_session(Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid}) -> - Node = node(SessPid), - case rpc:call(Node, emqttd_session, destroy, [SessPid, ClientId]) of - ok -> - remove_session(Session); - {badrpc, nodedown} -> - ?LOG(error, "Node '~s' down", [Node], Session), - remove_session(Session); - {badrpc, Reason} -> - ?LOG(error, "Failed to destory ~p on remote node ~p for ~s", - [SessPid, Node, Reason], Session), - {error, Reason} - end. - -remove_session(Session) -> - mnesia:dirty_delete_object(Session). - -monitor_session(ClientId, SessPid, State = #state{monitors = Monitors}) -> - MRef = erlang:monitor(process, SessPid), - State#state{monitors = dict:store(MRef, ClientId, Monitors)}. - -erase_monitor(MRef, State = #state{monitors = Monitors}) -> - erlang:demonitor(MRef, [flush]), - State#state{monitors = dict:erase(MRef, Monitors)}. diff --git a/src/emqttd_sm_helper.erl b/src/emqttd_sm_helper.erl deleted file mode 100644 index 7a1875be1..000000000 --- a/src/emqttd_sm_helper.erl +++ /dev/null @@ -1,89 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Session Helper. --module(emqttd_sm_helper). - --author("Feng Lee "). - --behaviour(gen_server). - --include("emqttd.hrl"). - --include("emqttd_internal.hrl"). - --include_lib("stdlib/include/ms_transform.hrl"). - -%% API Function Exports --export([start_link/1]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {stats_fun, ticker}). - --define(LOCK, {?MODULE, clean_sessions}). - -%% @doc Start a session helper --spec(start_link(fun()) -> {ok, pid()} | ignore | {error, term()}). -start_link(StatsFun) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [StatsFun], []). - -init([StatsFun]) -> - ekka:monitor(membership), - {ok, TRef} = timer:send_interval(timer:seconds(1), tick), - {ok, #state{stats_fun = StatsFun, ticker = TRef}}. - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info({membership, {mnesia, down, Node}}, State) -> - Fun = fun() -> - ClientIds = - mnesia:select(mqtt_session, [{#mqtt_session{client_id = '$1', sess_pid = '$2', _ = '_'}, - [{'==', {node, '$2'}, Node}], ['$1']}]), - lists:foreach(fun(ClientId) -> mnesia:delete({mqtt_session, ClientId}) end, ClientIds) - end, - global:trans({?LOCK, self()}, fun() -> mnesia:async_dirty(Fun) end), - {noreply, State, hibernate}; - -handle_info({membership, _Event}, State) -> - {noreply, State}; - -handle_info(tick, State) -> - {noreply, setstats(State), hibernate}; - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, _State = #state{ticker = TRef}) -> - timer:cancel(TRef), - ekka:unmonitor(membership). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -setstats(State = #state{stats_fun = StatsFun}) -> - StatsFun(ets:info(mqtt_local_session, size)), State. - diff --git a/src/emqttd_sm_sup.erl b/src/emqttd_sm_sup.erl deleted file mode 100644 index f26716e0d..000000000 --- a/src/emqttd_sm_sup.erl +++ /dev/null @@ -1,52 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Session Manager Supervisor. - --module(emqttd_sm_sup). - --behaviour(supervisor). - --author("Feng Lee "). - --include("emqttd.hrl"). - --define(HELPER, emqttd_sm_helper). - -%% API --export([start_link/0]). - -%% Supervisor callbacks --export([init/1]). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - %% Create session tables - ets:new(mqtt_local_session, [public, ordered_set, named_table, {write_concurrency, true}]), - - %% Helper - StatsFun = emqttd_stats:statsfun('sessions/count', 'sessions/max'), - Helper = {?HELPER, {?HELPER, start_link, [StatsFun]}, - permanent, 5000, worker, [?HELPER]}, - - %% SM Pool Sup - MFA = {emqttd_sm, start_link, []}, - PoolSup = emqttd_pool_sup:spec([emqttd_sm, hash, erlang:system_info(schedulers), MFA]), - - {ok, {{one_for_all, 10, 3600}, [Helper, PoolSup]}}. - diff --git a/src/emqttd_stats.erl b/src/emqttd_stats.erl deleted file mode 100644 index bfd60ea15..000000000 --- a/src/emqttd_stats.erl +++ /dev/null @@ -1,211 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_stats). - --behaviour(gen_server). - --author("Feng Lee "). - --include("emqttd.hrl"). - --export([start_link/0, stop/0]). - -%% Client and Session Stats --export([set_client_stats/2, get_client_stats/1, del_client_stats/1, - set_session_stats/2, get_session_stats/1, del_session_stats/1]). - -%% Statistics API. --export([statsfun/1, statsfun/2, getstats/0, getstat/1, setstat/2, setstats/3]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {tick}). - --type(stats() :: list({atom(), non_neg_integer()})). - --define(STATS_TAB, mqtt_stats). --define(CLIENT_STATS_TAB, mqtt_client_stats). --define(SESSION_STATS_TAB, mqtt_session_stats). - -%% $SYS Topics for Clients --define(SYSTOP_CLIENTS, [ - 'clients/count', % clients connected current - 'clients/max' % max clients connected -]). - -%% $SYS Topics for Sessions --define(SYSTOP_SESSIONS, [ - 'sessions/count', - 'sessions/max' -]). - -%% $SYS Topics for Subscribers --define(SYSTOP_PUBSUB, [ - 'topics/count', % ... - 'topics/max', % ... - 'subscribers/count', % ... - 'subscribers/max', % ... - 'subscriptions/count', % ... - 'subscriptions/max', % ... - 'routes/count', % ... - 'routes/max' % ... -]). - -%% $SYS Topic for retained --define(SYSTOP_RETAINED, [ - 'retained/count', - 'retained/max' -]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -%% @doc Start stats server --spec(start_link() -> {ok, pid()} | ignore | {error, term()}). -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -stop() -> - gen_server:call(?MODULE, stop). - --spec(set_client_stats(binary(), stats()) -> true). -set_client_stats(ClientId, Stats) -> - ets:insert(?CLIENT_STATS_TAB, {ClientId, [{'$ts', emqttd_time:now_secs()}|Stats]}). - --spec(get_client_stats(binary()) -> stats()). -get_client_stats(ClientId) -> - case ets:lookup(?CLIENT_STATS_TAB, ClientId) of - [{_, Stats}] -> Stats; - [] -> [] - end. - --spec(del_client_stats(binary()) -> true). -del_client_stats(ClientId) -> - ets:delete(?CLIENT_STATS_TAB, ClientId). - --spec(set_session_stats(binary(), stats()) -> true). -set_session_stats(ClientId, Stats) -> - ets:insert(?SESSION_STATS_TAB, {ClientId, [{'$ts', emqttd_time:now_secs()}|Stats]}). - --spec(get_session_stats(binary()) -> stats()). -get_session_stats(ClientId) -> - case ets:lookup(?SESSION_STATS_TAB, ClientId) of - [{_, Stats}] -> Stats; - [] -> [] - end. - --spec(del_session_stats(binary()) -> true). -del_session_stats(ClientId) -> - ets:delete(?SESSION_STATS_TAB, ClientId). - -%% @doc Generate stats fun --spec(statsfun(Stat :: atom()) -> fun()). -statsfun(Stat) -> - fun(Val) -> setstat(Stat, Val) end. - --spec(statsfun(Stat :: atom(), MaxStat :: atom()) -> fun()). -statsfun(Stat, MaxStat) -> - fun(Val) -> setstats(Stat, MaxStat, Val) end. - -%% @doc Get broker statistics --spec(getstats() -> [{atom(), non_neg_integer()}]). -getstats() -> - lists:sort(ets:tab2list(?STATS_TAB)). - -%% @doc Get stats by name --spec(getstat(atom()) -> non_neg_integer() | undefined). -getstat(Name) -> - case ets:lookup(?STATS_TAB, Name) of - [{Name, Val}] -> Val; - [] -> undefined - end. - -%% @doc Set broker stats --spec(setstat(Stat :: atom(), Val :: pos_integer()) -> boolean()). -setstat(Stat, Val) -> - ets:update_element(?STATS_TAB, Stat, {2, Val}). - -%% @doc Set stats with max --spec(setstats(Stat :: atom(), MaxStat :: atom(), Val :: pos_integer()) -> ok). -setstats(Stat, MaxStat, Val) -> - gen_server:cast(?MODULE, {setstats, Stat, MaxStat, Val}). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - emqttd_time:seed(), - lists:foreach( - fun(Tab) -> - Tab = ets:new(Tab, [set, public, named_table, {write_concurrency, true}]) - end, [?STATS_TAB, ?CLIENT_STATS_TAB, ?SESSION_STATS_TAB]), - Topics = ?SYSTOP_CLIENTS ++ ?SYSTOP_SESSIONS ++ ?SYSTOP_PUBSUB ++ ?SYSTOP_RETAINED, - ets:insert(?STATS_TAB, [{Topic, 0} || Topic <- Topics]), - % Tick to publish stats - {ok, #state{tick = emqttd_broker:start_tick(tick)}, hibernate}. - -handle_call(stop, _From, State) -> - {stop, normal, ok, State}; - -handle_call(_Request, _From, State) -> - {reply, error, State}. - -%% atomic -handle_cast({setstats, Stat, MaxStat, Val}, State) -> - MaxVal = ets:lookup_element(?STATS_TAB, MaxStat, 2), - if - Val > MaxVal -> - ets:update_element(?STATS_TAB, MaxStat, {2, Val}); - true -> ok - end, - ets:update_element(?STATS_TAB, Stat, {2, Val}), - {noreply, State}; - -handle_cast(_Msg, State) -> - {noreply, State}. - -%% Interval Tick. -handle_info(tick, State) -> - [publish(Stat, Val) || {Stat, Val} <- ets:tab2list(?STATS_TAB)], - {noreply, State, hibernate}; - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, #state{tick = TRef}) -> - emqttd_broker:stop_tick(TRef). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -publish(Stat, Val) -> - Msg = emqttd_message:make(stats, stats_topic(Stat), bin(Val)), - emqttd:publish(emqttd_message:set_flag(sys, Msg)). - -stats_topic(Stat) -> - emqttd_topic:systop(list_to_binary(lists:concat(['stats/', Stat]))). - -bin(I) when is_integer(I) -> list_to_binary(integer_to_list(I)). - diff --git a/src/emqttd_sup.erl b/src/emqttd_sup.erl deleted file mode 100644 index 0d0bf496c..000000000 --- a/src/emqttd_sup.erl +++ /dev/null @@ -1,55 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_sup). - --behaviour(supervisor). - --author("Feng Lee "). - --include("emqttd.hrl"). - -%% API --export([start_link/0, start_child/1, start_child/2]). - -%% Supervisor callbacks --export([init/1]). - -%% Helper macro for declaring children of supervisor --define(CHILD(Mod, Type), {Mod, {Mod, start_link, []}, - permanent, 5000, Type, [Mod]}). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -start_child(ChildSpec) when is_tuple(ChildSpec) -> - supervisor:start_child(?MODULE, ChildSpec). - --spec(start_child(Mod::atom(), Type :: worker | supervisor) -> {ok, pid()}). -start_child(Mod, Type) when is_atom(Mod) and is_atom(Type) -> - supervisor:start_child(?MODULE, ?CHILD(Mod, Type)). - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - -init([]) -> - {ok, {{one_for_all, 0, 1}, []}}. - diff --git a/src/emqttd_sysmon.erl b/src/emqttd_sysmon.erl deleted file mode 100644 index 8a9489c9e..000000000 --- a/src/emqttd_sysmon.erl +++ /dev/null @@ -1,176 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc VM System Monitor --module(emqttd_sysmon). - --author("Feng Lee "). - --behavior(gen_server). - --include("emqttd_internal.hrl"). - --export([start_link/1]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {tickref, events = [], tracelog}). - --define(LOG_FMT, [{formatter_config, [time, " ", message, "\n"]}]). - --define(LOG(Msg, ProcInfo), - lager:warning([{sysmon, true}], "~s~n~p", [WarnMsg, ProcInfo])). - --define(LOG(Msg, ProcInfo, PortInfo), - lager:warning([{sysmon, true}], "~s~n~p~n~p", [WarnMsg, ProcInfo, PortInfo])). - -%% @doc Start system monitor --spec(start_link(Opts :: list(tuple())) -> - {ok, pid()} | ignore | {error, term()}). -start_link(Opts) -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [Opts], []). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([Opts]) -> - erlang:system_monitor(self(), parse_opt(Opts)), - {ok, TRef} = timer:send_interval(timer:seconds(1), reset), - %%TODO: don't trace for performance issue. - %%{ok, TraceLog} = start_tracelog(proplists:get_value(logfile, Opts)), - {ok, #state{tickref = TRef}}. - -parse_opt(Opts) -> - parse_opt(Opts, []). -parse_opt([], Acc) -> - Acc; -parse_opt([{long_gc, false}|Opts], Acc) -> - parse_opt(Opts, Acc); -parse_opt([{long_gc, Ms}|Opts], Acc) when is_integer(Ms) -> - parse_opt(Opts, [{long_gc, Ms}|Acc]); -parse_opt([{long_schedule, false}|Opts], Acc) -> - parse_opt(Opts, Acc); -parse_opt([{long_schedule, Ms}|Opts], Acc) when is_integer(Ms) -> - parse_opt(Opts, [{long_schedule, Ms}|Acc]); -parse_opt([{large_heap, Size}|Opts], Acc) when is_integer(Size) -> - parse_opt(Opts, [{large_heap, Size}|Acc]); -parse_opt([{busy_port, true}|Opts], Acc) -> - parse_opt(Opts, [busy_port|Acc]); -parse_opt([{busy_port, false}|Opts], Acc) -> - parse_opt(Opts, Acc); -parse_opt([{busy_dist_port, true}|Opts], Acc) -> - parse_opt(Opts, [busy_dist_port|Acc]); -parse_opt([{busy_dist_port, false}|Opts], Acc) -> - parse_opt(Opts, Acc); -parse_opt([_Opt|Opts], Acc) -> - parse_opt(Opts, Acc). - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info({monitor, Pid, long_gc, Info}, State) -> - suppress({long_gc, Pid}, fun() -> - WarnMsg = io_lib:format("long_gc warning: pid = ~p, info: ~p", [Pid, Info]), - ?LOG(WarnMsg, procinfo(Pid)), - publish(long_gc, WarnMsg) - end, State); - -handle_info({monitor, Pid, long_schedule, Info}, State) when is_pid(Pid) -> - suppress({long_schedule, Pid}, fun() -> - WarnMsg = io_lib:format("long_schedule warning: pid = ~p, info: ~p", [Pid, Info]), - ?LOG(WarnMsg, procinfo(Pid)), - publish(long_schedule, WarnMsg) - end, State); - -handle_info({monitor, Port, long_schedule, Info}, State) when is_port(Port) -> - suppress({long_schedule, Port}, fun() -> - WarnMsg = io_lib:format("long_schedule warning: port = ~p, info: ~p", [Port, Info]), - ?LOG(WarnMsg, erlang:port_info(Port)), - publish(long_schedule, WarnMsg) - end, State); - -handle_info({monitor, Pid, large_heap, Info}, State) -> - suppress({large_heap, Pid}, fun() -> - WarnMsg = io_lib:format("large_heap warning: pid = ~p, info: ~p", [Pid, Info]), - ?LOG(WarnMsg, procinfo(Pid)), - publish(large_heap, WarnMsg) - end, State); - -handle_info({monitor, SusPid, busy_port, Port}, State) -> - suppress({busy_port, Port}, fun() -> - WarnMsg = io_lib:format("busy_port warning: suspid = ~p, port = ~p", [SusPid, Port]), - ?LOG(WarnMsg, procinfo(SusPid), erlang:port_info(Port)), - publish(busy_port, WarnMsg) - end, State); - -handle_info({monitor, SusPid, busy_dist_port, Port}, State) -> - suppress({busy_dist_port, Port}, fun() -> - WarnMsg = io_lib:format("busy_dist_port warning: suspid = ~p, port = ~p", [SusPid, Port]), - ?LOG(WarnMsg, procinfo(SusPid), erlang:port_info(Port)), - publish(busy_dist_port, WarnMsg) - end, State); - -handle_info(reset, State) -> - {noreply, State#state{events = []}, hibernate}; - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, #state{tickref = TRef, tracelog = TraceLog}) -> - timer:cancel(TRef), - cancel_tracelog(TraceLog). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -suppress(Key, SuccFun, State = #state{events = Events}) -> - case lists:member(Key, Events) of - true -> - {noreply, State}; - false -> - SuccFun(), - {noreply, State#state{events = [Key|Events]}} - end. - -procinfo(Pid) -> - case {emqttd_vm:get_process_info(Pid), emqttd_vm:get_process_gc(Pid)} of - {undefined, _} -> undefined; - {_, undefined} -> undefined; - {Info, GcInfo} -> Info ++ GcInfo - end. - -publish(Sysmon, WarnMsg) -> - Msg = emqttd_message:make(sysmon, topic(Sysmon), iolist_to_binary(WarnMsg)), - emqttd:publish(emqttd_message:set_flag(sys, Msg)). - -topic(Sysmon) -> - emqttd_topic:systop(list_to_binary(lists:concat(['sysmon/', Sysmon]))). - -%% start_tracelog(undefined) -> -%% {ok, undefined}; -%% start_tracelog(LogFile) -> -%% lager:trace_file(LogFile, [{sysmon, true}], info, ?LOG_FMT). - -cancel_tracelog(undefined) -> - ok; -cancel_tracelog(TraceLog) -> - lager:stop_trace(TraceLog). - diff --git a/src/emqttd_trace.erl b/src/emqttd_trace.erl deleted file mode 100644 index b87359416..000000000 --- a/src/emqttd_trace.erl +++ /dev/null @@ -1,117 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @docTrace MQTT packets/messages by ClientID or Topic. --module(emqttd_trace). - --behaviour(gen_server). - --author("Feng Lee "). - --include("emqttd_internal.hrl"). - -%% API Function Exports --export([start_link/0]). - --export([start_trace/2, stop_trace/1, all_traces/0]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {level, traces}). - --type(trace_who() :: {client | topic, binary()}). - --define(TRACE_OPTIONS, [{formatter_config, [time, " [",severity,"] ", message, "\n"]}]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - --spec(start_link() -> {ok, pid()}). -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -%% @doc Start to trace client or topic. --spec(start_trace(trace_who(), string()) -> ok | {error, term()}). -start_trace({client, ClientId}, LogFile) -> - start_trace({start_trace, {client, ClientId}, LogFile}); - -start_trace({topic, Topic}, LogFile) -> - start_trace({start_trace, {topic, Topic}, LogFile}). - -start_trace(Req) -> gen_server:call(?MODULE, Req, infinity). - -%% @doc Stop tracing client or topic. --spec(stop_trace(trace_who()) -> ok | {error, term()}). -stop_trace({client, ClientId}) -> - gen_server:call(?MODULE, {stop_trace, {client, ClientId}}); -stop_trace({topic, Topic}) -> - gen_server:call(?MODULE, {stop_trace, {topic, Topic}}). - -%% @doc Lookup all traces. --spec(all_traces() -> [{Who :: trace_who(), LogFile :: string()}]). -all_traces() -> gen_server:call(?MODULE, all_traces). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - {ok, #state{level = debug, traces = #{}}}. - -handle_call({start_trace, Who, LogFile}, _From, State = #state{level = Level, traces = Traces}) -> - case lager:trace_file(LogFile, [Who], Level, ?TRACE_OPTIONS) of - {ok, exists} -> - {reply, {error, existed}, State}; - {ok, Trace} -> - {reply, ok, State#state{traces = maps:put(Who, {Trace, LogFile}, Traces)}}; - {error, Error} -> - {reply, {error, Error}, State} - end; - -handle_call({stop_trace, Who}, _From, State = #state{traces = Traces}) -> - case maps:find(Who, Traces) of - {ok, {Trace, _LogFile}} -> - case lager:stop_trace(Trace) of - ok -> ok; - {error, Error} -> lager:error("Stop trace ~p error: ~p", [Who, Error]) - end, - {reply, ok, State#state{traces = maps:remove(Who, Traces)}}; - error -> - {reply, {error, not_found}, State} - end; - -handle_call(all_traces, _From, State = #state{traces = Traces}) -> - {reply, [{Who, LogFile} || {Who, {_Trace, LogFile}} - <- maps:to_list(Traces)], State}; - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - diff --git a/src/emqttd_ws.erl b/src/emqttd_ws.erl deleted file mode 100644 index e2375e4a6..000000000 --- a/src/emqttd_ws.erl +++ /dev/null @@ -1,120 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_ws). - --author("Feng Lee "). - --include("emqttd_protocol.hrl"). - --import(proplists, [get_value/3]). - --export([handle_request/1, ws_loop/3]). - -%% WebSocket Loop State --record(wsocket_state, {peername, client_pid, max_packet_size, parser}). - --define(WSLOG(Level, Format, Args, State), - lager:Level("WsClient(~s): " ++ Format, - [esockd_net:format(State#wsocket_state.peername) | Args])). - - -handle_request(Req) -> - handle_request(Req:get(method), Req:get(path), Req). - -%%-------------------------------------------------------------------- -%% MQTT Over WebSocket -%%-------------------------------------------------------------------- - -handle_request('GET', "/mqtt", Req) -> - lager:debug("WebSocket Connection from: ~s", [Req:get(peer)]), - Upgrade = Req:get_header_value("Upgrade"), - Proto = check_protocol_header(Req), - case {is_websocket(Upgrade), Proto} of - {true, "mqtt" ++ _Vsn} -> - case Req:get(peername) of - {ok, Peername} -> - {ok, ProtoEnv} = emqttd:env(protocol), - PacketSize = get_value(max_packet_size, ProtoEnv, ?MAX_PACKET_SIZE), - Parser = emqttd_parser:initial_state(PacketSize), - %% Upgrade WebSocket. - {ReentryWs, ReplyChannel} = mochiweb_websocket:upgrade_connection(Req, fun ?MODULE:ws_loop/3), - {ok, ClientPid} = emqttd_ws_client_sup:start_client(self(), Req, ReplyChannel), - ReentryWs(#wsocket_state{peername = Peername, - parser = Parser, - max_packet_size = PacketSize, - client_pid = ClientPid}); - {error, Reason} -> - lager:error("Get peername with error ~s", [Reason]), - Req:respond({400, [], <<"Bad Request">>}) - end; - {false, _} -> - lager:error("Not WebSocket: Upgrade = ~s", [Upgrade]), - Req:respond({400, [], <<"Bad Request">>}); - {_, Proto} -> - lager:error("WebSocket with error Protocol: ~s", [Proto]), - Req:respond({400, [], <<"Bad WebSocket Protocol">>}) - end; - -handle_request(Method, Path, Req) -> - lager:error("Unexpected WS Request: ~s ~s", [Method, Path]), - Req:not_found(). - -is_websocket(Upgrade) -> - Upgrade =/= undefined andalso string:to_lower(Upgrade) =:= "websocket". - -check_protocol_header(Req) -> - case emqttd:env(websocket_protocol_header, false) of - true -> get_protocol_header(Req); - false -> "mqtt-v3.1.1" - end. - -get_protocol_header(Req) -> - case Req:get_header_value("EMQ-WebSocket-Protocol") of - undefined -> Req:get_header_value("Sec-WebSocket-Protocol"); - Proto -> Proto - end. - -%%-------------------------------------------------------------------- -%% Receive Loop -%%-------------------------------------------------------------------- - -%% @doc WebSocket frame receive loop. -ws_loop(<<>>, State, _ReplyChannel) -> - State; -ws_loop([<<>>], State, _ReplyChannel) -> - State; -ws_loop(Data, State = #wsocket_state{client_pid = ClientPid, parser = Parser}, ReplyChannel) -> - ?WSLOG(debug, "RECV ~p", [Data], State), - emqttd_metrics:inc('bytes/received', iolist_size(Data)), - case catch emqttd_parser:parse(iolist_to_binary(Data), Parser) of - {more, NewParser} -> - State#wsocket_state{parser = NewParser}; - {ok, Packet, Rest} -> - gen_server:cast(ClientPid, {received, Packet}), - ws_loop(Rest, reset_parser(State), ReplyChannel); - {error, Error} -> - ?WSLOG(error, "Frame error: ~p", [Error], State), - exit({shutdown, Error}); - {'EXIT', Reason} -> - ?WSLOG(error, "Frame error: ~p", [Reason], State), - ?WSLOG(error, "Error data: ~p", [Data], State), - exit({shutdown, parser_error}) - end. - -reset_parser(State = #wsocket_state{max_packet_size = PacketSize}) -> - State#wsocket_state{parser = emqttd_parser:initial_state(PacketSize)}. - diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl deleted file mode 100644 index 0462e3220..000000000 --- a/src/emqttd_ws_client.erl +++ /dev/null @@ -1,327 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc MQTT WebSocket Connection. - --module(emqttd_ws_client). - --behaviour(gen_server2). - --author("Feng Lee "). - --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --include("emqttd_internal.hrl"). - --import(proplists, [get_value/3]). - -%% API Exports --export([start_link/4]). - -%% Management and Monitor API --export([info/1, stats/1, kick/1, clean_acl_cache/2]). - -%% SUB/UNSUB Asynchronously --export([subscribe/2, unsubscribe/2]). - -%% Get the session proc? --export([session/1]). - -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - -%% gen_server2 Callbacks --export([prioritise_call/4, prioritise_info/3, handle_pre_hibernate/1]). - -%% WebSocket Client State --record(wsclient_state, {ws_pid, peername, connection, proto_state, keepalive, - enable_stats, force_gc_count}). - --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). - --define(WSLOG(Level, Format, Args, State), - lager:Level("WsClient(~s): " ++ Format, - [esockd_net:format(State#wsclient_state.peername) | Args])). - -%% @doc Start WebSocket Client. -start_link(Env, WsPid, Req, ReplyChannel) -> - gen_server2:start_link(?MODULE, [Env, WsPid, Req, ReplyChannel], - [{spawn_opt, ?FULLSWEEP_OPTS}]). %% Tune GC. - -info(CPid) -> - gen_server2:call(CPid, info). - -stats(CPid) -> - gen_server2:call(CPid, stats). - -kick(CPid) -> - gen_server2:call(CPid, kick). - -subscribe(CPid, TopicTable) -> - CPid ! {subscribe, TopicTable}. - -unsubscribe(CPid, Topics) -> - CPid ! {unsubscribe, Topics}. - -session(CPid) -> - gen_server2:call(CPid, session). - -clean_acl_cache(CPid, Topic) -> - gen_server2:call(CPid, {clean_acl_cache, Topic}). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([Env, WsPid, Req, ReplyChannel]) -> - process_flag(trap_exit, true), - Conn = Req:get(connection), - true = link(WsPid), - case Req:get(peername) of - {ok, Peername} -> - Headers = mochiweb_headers:to_list( - mochiweb_request:get(headers, Req)), - ProtoState = emqttd_protocol:init(Conn, Peername, send_fun(ReplyChannel), - [{ws_initial_headers, Headers} | Env]), - IdleTimeout = get_value(client_idle_timeout, Env, 30000), - EnableStats = get_value(client_enable_stats, Env, false), - ForceGcCount = emqttd_gc:conn_max_gc_count(), - {ok, #wsclient_state{connection = Conn, - ws_pid = WsPid, - peername = Peername, - proto_state = ProtoState, - enable_stats = EnableStats, - force_gc_count = ForceGcCount}, - IdleTimeout, {backoff, 2000, 2000, 20000}, ?MODULE}; - {error, enotconn} -> Conn:fast_close(), - exit(WsPid, normal), - exit(normal); - {error, Reason} -> Conn:fast_close(), - exit(WsPid, normal), - exit({shutdown, Reason}) - end. - -prioritise_call(Msg, _From, _Len, _State) -> - case Msg of info -> 10; stats -> 10; state -> 10; _ -> 5 end. - -prioritise_info(Msg, _Len, _State) -> - case Msg of {redeliver, _} -> 5; _ -> 0 end. - -handle_pre_hibernate(State = #wsclient_state{ws_pid = WsPid}) -> - erlang:garbage_collect(WsPid), - {hibernate, emqttd_gc:reset_conn_gc_count(#wsclient_state.force_gc_count, emit_stats(State))}. - -handle_call(info, From, State = #wsclient_state{peername = Peername, - proto_state = ProtoState}) -> - Info = [{websocket, true}, {peername, Peername} | emqttd_protocol:info(ProtoState)], - {reply, Stats, _, _} = handle_call(stats, From, State), - reply(lists:append(Info, Stats), State); - -handle_call(stats, _From, State = #wsclient_state{proto_state = ProtoState}) -> - reply(lists:append([emqttd_misc:proc_stats(), - wsock_stats(State), - emqttd_protocol:stats(ProtoState)]), State); - -handle_call(kick, _From, State) -> - {stop, {shutdown, kick}, ok, State}; - -handle_call(session, _From, State = #wsclient_state{proto_state = ProtoState}) -> - reply(emqttd_protocol:session(ProtoState), State); - -handle_call({clean_acl_cache, Topic}, _From, State) -> - erase({acl, publish, Topic}), - reply(ok, State); - -handle_call(Req, _From, State) -> - ?WSLOG(error, "Unexpected request: ~p", [Req], State), - reply({error, unexpected_request}, State). - -handle_cast({received, Packet}, State = #wsclient_state{proto_state = ProtoState}) -> - emqttd_metrics:received(Packet), - case emqttd_protocol:received(Packet, ProtoState) of - {ok, ProtoState1} -> - {noreply, gc(State#wsclient_state{proto_state = ProtoState1}), hibernate}; - {error, Error} -> - ?WSLOG(error, "Protocol error - ~p", [Error], State), - shutdown(Error, State); - {error, Error, ProtoState1} -> - shutdown(Error, State#wsclient_state{proto_state = ProtoState1}); - {stop, Reason, ProtoState1} -> - stop(Reason, State#wsclient_state{proto_state = ProtoState1}) - end; - -handle_cast(Msg, State) -> - ?WSLOG(error, "Unexpected Msg: ~p", [Msg], State), - {noreply, State, hibernate}. - -handle_info({subscribe, TopicTable}, State) -> - with_proto( - fun(ProtoState) -> - emqttd_protocol:subscribe(TopicTable, ProtoState) - end, State); - -handle_info({unsubscribe, Topics}, State) -> - with_proto( - fun(ProtoState) -> - emqttd_protocol:unsubscribe(Topics, ProtoState) - end, State); - -handle_info({suback, PacketId, GrantedQos}, State) -> - with_proto( - fun(ProtoState) -> - Packet = ?SUBACK_PACKET(PacketId, GrantedQos), - emqttd_protocol:send(Packet, ProtoState) - end, State); - -handle_info({deliver, Message}, State) -> - with_proto( - fun(ProtoState) -> - emqttd_protocol:send(Message, ProtoState) - end, gc(State)); - -handle_info({redeliver, {?PUBREL, PacketId}}, State) -> - with_proto( - fun(ProtoState) -> - emqttd_protocol:pubrel(PacketId, ProtoState) - end, State); - -handle_info(emit_stats, State) -> - {noreply, emit_stats(State), hibernate}; - -handle_info(timeout, State) -> - shutdown(idle_timeout, State); - -handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> - ?WSLOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid], State), - shutdown(conflict, State); - -handle_info({shutdown, Reason}, State) -> - shutdown(Reason, State); - -handle_info({keepalive, start, Interval}, State = #wsclient_state{connection = Conn}) -> - ?WSLOG(debug, "Keepalive at the interval of ~p", [Interval], State), - case emqttd_keepalive:start(stat_fun(Conn), Interval, {keepalive, check}) of - {ok, KeepAlive} -> - {noreply, State#wsclient_state{keepalive = KeepAlive}, hibernate}; - {error, Error} -> - ?WSLOG(warning, "Keepalive error - ~p", [Error], State), - shutdown(Error, State) - end; - -handle_info({keepalive, check}, State = #wsclient_state{keepalive = KeepAlive}) -> - case emqttd_keepalive:check(KeepAlive) of - {ok, KeepAlive1} -> - {noreply, emit_stats(State#wsclient_state{keepalive = KeepAlive1}), hibernate}; - {error, timeout} -> - ?WSLOG(debug, "Keepalive Timeout!", [], State), - shutdown(keepalive_timeout, State); - {error, Error} -> - ?WSLOG(warning, "Keepalive error - ~p", [Error], State), - shutdown(keepalive_error, State) - end; - -handle_info({'EXIT', WsPid, normal}, State = #wsclient_state{ws_pid = WsPid}) -> - stop(normal, State); - -handle_info({'EXIT', WsPid, Reason}, State = #wsclient_state{ws_pid = WsPid}) -> - ?WSLOG(error, "shutdown: ~p",[Reason], State), - shutdown(Reason, State); - -%% The session process exited unexpectedly. -handle_info({'EXIT', Pid, Reason}, State = #wsclient_state{proto_state = ProtoState}) -> - case emqttd_protocol:session(ProtoState) of - Pid -> stop(Reason, State); - _ -> ?WSLOG(error, "Unexpected EXIT: ~p, Reason: ~p", [Pid, Reason], State), - {noreply, State, hibernate} - end; - -handle_info(Info, State) -> - ?WSLOG(error, "Unexpected Info: ~p", [Info], State), - {noreply, State, hibernate}. - -terminate(Reason, #wsclient_state{proto_state = ProtoState, keepalive = KeepAlive}) -> - emqttd_keepalive:cancel(KeepAlive), - case Reason of - {shutdown, Error} -> - emqttd_protocol:shutdown(Error, ProtoState); - _ -> - emqttd_protocol:shutdown(Reason, ProtoState) - end. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -send_fun(ReplyChannel) -> - Self = self(), - fun(Packet) -> - Data = emqttd_serializer:serialize(Packet), - emqttd_metrics:inc('bytes/sent', iolist_size(Data)), - case ReplyChannel({binary, Data}) of - ok -> ok; - {error, Reason} -> Self ! {shutdown, Reason} - end - end. - -stat_fun(Conn) -> - fun() -> - case Conn:getstat([recv_oct]) of - {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; - {error, Error} -> {error, Error} - end - end. - -emit_stats(State = #wsclient_state{proto_state = ProtoState}) -> - emit_stats(emqttd_protocol:clientid(ProtoState), State). - -emit_stats(_ClientId, State = #wsclient_state{enable_stats = false}) -> - State; -emit_stats(undefined, State) -> - State; -emit_stats(ClientId, State) -> - {reply, Stats, _, _} = handle_call(stats, undefined, State), - emqttd_stats:set_client_stats(ClientId, Stats), - State. - -wsock_stats(#wsclient_state{connection = Conn}) -> - case Conn:getstat(?SOCK_STATS) of - {ok, Ss} -> Ss; - {error, _} -> [] - end. - -with_proto(Fun, State = #wsclient_state{proto_state = ProtoState}) -> - {ok, ProtoState1} = Fun(ProtoState), - {noreply, State#wsclient_state{proto_state = ProtoState1}, hibernate}. - -reply(Reply, State) -> - {reply, Reply, State, hibernate}. - -shutdown(Reason, State) -> - stop({shutdown, Reason}, State). - -stop(Reason, State) -> - {stop, Reason, State}. - -gc(State) -> - Cb = fun() -> emit_stats(State) end, - emqttd_gc:maybe_force_gc(#wsclient_state.force_gc_count, State, Cb). - diff --git a/src/emqttd_ws_client_sup.erl b/src/emqttd_ws_client_sup.erl deleted file mode 100644 index 48f3b1193..000000000 --- a/src/emqttd_ws_client_sup.erl +++ /dev/null @@ -1,46 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_ws_client_sup). - --author("Feng Lee "). - --behavior(supervisor). - --export([start_link/0, start_client/3]). - --export([init/1]). - -%% @doc Start websocket client supervisor --spec(start_link() -> {ok, pid()}). -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -%% @doc Start a WebSocket Connection. --spec(start_client(pid(), mochiweb_request:request(), fun()) -> {ok, pid()}). -start_client(WsPid, Req, ReplyChannel) -> - supervisor:start_child(?MODULE, [WsPid, Req, ReplyChannel]). - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - -init([]) -> - Env = lists:append(emqttd:env(client, []), emqttd:env(protocol, [])), - {ok, {{simple_one_for_one, 0, 1}, - [{ws_client, {emqttd_ws_client, start_link, [Env]}, - temporary, 5000, worker, [emqttd_ws_client]}]}}. - diff --git a/src/emqx.app.src b/src/emqx.app.src new file mode 100644 index 000000000..ce643634e --- /dev/null +++ b/src/emqx.app.src @@ -0,0 +1,12 @@ +{application,emqx, + [{description,"EMQ X Broker"}, + {vsn,"git"}, + {modules,[]}, + {registered,[emqx_sup]}, + {applications,[kernel,stdlib,jsx,gproc,gen_rpc,esockd, + cowboy]}, + {env,[]}, + {mod,{emqx_app,[]}}, + {maintainers,["Feng Lee "]}, + {licenses,["Apache-2.0"]}, + {links,[{"Github","https://github.com/emqx/emqx"}]}]}. diff --git a/src/emqx.erl b/src/emqx.erl new file mode 100644 index 000000000..76e966a59 --- /dev/null +++ b/src/emqx.erl @@ -0,0 +1,160 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx). + +-include("emqx.hrl"). + +%% Start/Stop the application +-export([start/0, is_running/1, stop/0]). + +%% PubSub API +-export([subscribe/1, subscribe/2, subscribe/3]). +-export([publish/1]). +-export([unsubscribe/1]). + +%% PubSub management API +-export([topics/0, subscriptions/1, subscribers/1, subscribed/2]). + +%% Hooks API +-export([hook/2, hook/3, hook/4, unhook/2, run_hooks/2, run_hooks/3]). + +%% Shutdown and reboot +-export([shutdown/0, shutdown/1, reboot/0]). + +-define(APP, ?MODULE). + +%%------------------------------------------------------------------------------ +%% Bootstrap, is_running... +%%------------------------------------------------------------------------------ + +%% @doc Start emqx application +-spec(start() -> {ok, list(atom())} | {error, term()}). +start() -> + %% Check OS + %% Check VM + %% Check Mnesia + application:ensure_all_started(?APP). + +%% @doc Stop emqx application. +-spec(stop() -> ok | {error, term()}). +stop() -> + application:stop(?APP). + +%% @doc Is emqx running? +-spec(is_running(node()) -> boolean()). +is_running(Node) -> + case rpc:call(Node, erlang, whereis, [?APP]) of + {badrpc, _} -> false; + undefined -> false; + Pid when is_pid(Pid) -> true + end. + +%%------------------------------------------------------------------------------ +%% PubSub API +%%------------------------------------------------------------------------------ + +-spec(subscribe(emqx_topic:topic() | string()) -> ok). +subscribe(Topic) -> + emqx_broker:subscribe(iolist_to_binary(Topic)). + +-spec(subscribe(emqx_topic:topic() | string(), emqx_types:subid() | emqx_types:subopts()) -> ok). +subscribe(Topic, SubId) when is_atom(SubId); is_binary(SubId)-> + emqx_broker:subscribe(iolist_to_binary(Topic), SubId); +subscribe(Topic, SubOpts) when is_map(SubOpts) -> + emqx_broker:subscribe(iolist_to_binary(Topic), SubOpts). + +-spec(subscribe(emqx_topic:topic() | string(), + emqx_types:subid() | pid(), emqx_types:subopts()) -> ok). +subscribe(Topic, SubId, SubOpts) when (is_atom(SubId) orelse is_binary(SubId)), is_map(SubOpts) -> + emqx_broker:subscribe(iolist_to_binary(Topic), SubId, SubOpts). + +-spec(publish(emqx_types:message()) -> emqx_types:deliver_results()). +publish(Msg) -> + emqx_broker:publish(Msg). + +-spec(unsubscribe(emqx_topic:topic() | string()) -> ok). +unsubscribe(Topic) -> + emqx_broker:unsubscribe(iolist_to_binary(Topic)). + +%%------------------------------------------------------------------------------ +%% PubSub management API +%%------------------------------------------------------------------------------ + +-spec(topics() -> list(emqx_topic:topic())). +topics() -> emqx_router:topics(). + +-spec(subscribers(emqx_topic:topic() | string()) -> list(emqx_types:subscriber())). +subscribers(Topic) -> + emqx_broker:subscribers(iolist_to_binary(Topic)). + +-spec(subscriptions(pid()) -> [{emqx_topic:topic(), emqx_types:subopts()}]). +subscriptions(SubPid) when is_pid(SubPid) -> + emqx_broker:subscriptions(SubPid). + +-spec(subscribed(pid() | emqx_types:subid(), emqx_topic:topic() | string()) -> boolean()). +subscribed(SubPid, Topic) when is_pid(SubPid) -> + emqx_broker:subscribed(SubPid, iolist_to_binary(Topic)); +subscribed(SubId, Topic) when is_atom(SubId); is_binary(SubId) -> + emqx_broker:subscribed(SubId, iolist_to_binary(Topic)). + +%%------------------------------------------------------------------------------ +%% Hooks API +%%------------------------------------------------------------------------------ + +-spec(hook(emqx_hooks:hookpoint(), emqx_hooks:action()) -> ok | {error, already_exists}). +hook(HookPoint, Action) -> + emqx_hooks:add(HookPoint, Action). + +-spec(hook(emqx_hooks:hookpoint(), emqx_hooks:action(), emqx_hooks:filter() | integer()) + -> ok | {error, already_exists}). +hook(HookPoint, Action, Priority) when is_integer(Priority) -> + emqx_hooks:add(HookPoint, Action, Priority); +hook(HookPoint, Action, Filter) when is_function(Filter); is_tuple(Filter) -> + emqx_hooks:add(HookPoint, Action, Filter); +hook(HookPoint, Action, InitArgs) when is_list(InitArgs) -> + emqx_hooks:add(HookPoint, Action, InitArgs). + +-spec(hook(emqx_hooks:hookpoint(), emqx_hooks:action(), emqx_hooks:filter(), integer()) + -> ok | {error, already_exists}). +hook(HookPoint, Action, Filter, Priority) -> + emqx_hooks:add(HookPoint, Action, Filter, Priority). + +-spec(unhook(emqx_hooks:hookpoint(), emqx_hooks:action()) -> ok). +unhook(HookPoint, Action) -> + emqx_hooks:del(HookPoint, Action). + +-spec(run_hooks(emqx_hooks:hookpoint(), list(any())) -> ok | stop). +run_hooks(HookPoint, Args) -> + emqx_hooks:run(HookPoint, Args). + +-spec(run_hooks(emqx_hooks:hookpoint(), list(any()), any()) -> {ok | stop, any()}). +run_hooks(HookPoint, Args, Acc) -> + emqx_hooks:run(HookPoint, Args, Acc). + +%%------------------------------------------------------------------------------ +%% Shutdown and reboot +%%------------------------------------------------------------------------------ + +shutdown() -> + shutdown(normal). + +shutdown(Reason) -> + emqx_logger:error("emqx shutdown for ~s", [Reason]), + emqx_plugins:unload(), + lists:foreach(fun application:stop/1, [emqx, ekka, cowboy, ranch, esockd, gproc]). + +reboot() -> + lists:foreach(fun application:start/1, [gproc, esockd, ranch, cowboy, ekka, emqx]). + diff --git a/src/emqx_access_control.erl b/src/emqx_access_control.erl new file mode 100644 index 000000000..1c43eb746 --- /dev/null +++ b/src/emqx_access_control.erl @@ -0,0 +1,209 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_access_control). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/0]). +-export([authenticate/2]). +-export([check_acl/3, reload_acl/0]). +-export([register_mod/3, register_mod/4, unregister_mod/2]). +-export([lookup_mods/1]). +-export([stop/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(TAB, ?MODULE). +-define(SERVER, ?MODULE). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% @doc Start access control server. +-spec(start_link() -> {ok, pid()} | {error, term()}). +start_link() -> + start_with(fun register_default_acl/0). + +start_with(Fun) -> + case gen_server:start_link({local, ?SERVER}, ?MODULE, [], []) of + {ok, Pid} -> + Fun(), {ok, Pid}; + {error, Reason} -> + {error, Reason} + end. + +register_default_acl() -> + case emqx_config:get_env(acl_file) of + undefined -> ok; + File -> register_mod(acl, emqx_acl_internal, [File]) + end. + +-spec(authenticate(emqx_types:credentials(), emqx_types:password()) + -> ok | {ok, map()} | {continue, map()} | {error, term()}). +authenticate(Credentials, Password) -> + authenticate(Credentials, Password, lookup_mods(auth)). + +authenticate(Credentials, _Password, []) -> + Zone = maps:get(zone, Credentials, undefined), + case emqx_zone:get_env(Zone, allow_anonymous, false) of + true -> ok; + false -> {error, auth_modules_not_found} + end; + +authenticate(Credentials, Password, [{Mod, State, _Seq} | Mods]) -> + case catch Mod:check(Credentials, Password, State) of + ok -> ok; + {ok, IsSuper} when is_boolean(IsSuper) -> + {ok, #{is_superuser => IsSuper}}; + {ok, Result} when is_map(Result) -> + {ok, Result}; + {continue, Result} when is_map(Result) -> + {continue, Result}; + ignore -> + authenticate(Credentials, Password, Mods); + {error, Reason} -> + {error, Reason}; + {'EXIT', Error} -> + {error, Error} + end. + +%% @doc Check ACL +-spec(check_acl(emqx_types:credentials(), emqx_types:pubsub(), emqx_types:topic()) -> allow | deny). +check_acl(Credentials, PubSub, Topic) when PubSub =:= publish; PubSub =:= subscribe -> + check_acl(Credentials, PubSub, Topic, lookup_mods(acl), emqx_acl_cache:is_enabled()). + +check_acl(Credentials, PubSub, Topic, AclMods, false) -> + do_check_acl(Credentials, PubSub, Topic, AclMods); +check_acl(Credentials, PubSub, Topic, AclMods, true) -> + case emqx_acl_cache:get_acl_cache(PubSub, Topic) of + not_found -> + AclResult = do_check_acl(Credentials, PubSub, Topic, AclMods), + emqx_acl_cache:put_acl_cache(PubSub, Topic, AclResult), + AclResult; + AclResult -> + AclResult + end. + +do_check_acl(#{zone := Zone}, _PubSub, _Topic, []) -> + emqx_zone:get_env(Zone, acl_nomatch, deny); +do_check_acl(Credentials, PubSub, Topic, [{Mod, State, _Seq}|AclMods]) -> + case Mod:check_acl({Credentials, PubSub, Topic}, State) of + allow -> allow; + deny -> deny; + ignore -> do_check_acl(Credentials, PubSub, Topic, AclMods) + end. + +-spec(reload_acl() -> list(ok | {error, term()})). +reload_acl() -> + [Mod:reload_acl(State) || {Mod, State, _Seq} <- lookup_mods(acl)]. + +%% @doc Register an Auth/ACL module. +-spec(register_mod(auth | acl, module(), list()) -> ok | {error, term()}). +register_mod(Type, Mod, Opts) when Type =:= auth; Type =:= acl -> + register_mod(Type, Mod, Opts, 0). + +-spec(register_mod(auth | acl, module(), list(), non_neg_integer()) + -> ok | {error, term()}). +register_mod(Type, Mod, Opts, Seq) when Type =:= auth; Type =:= acl-> + gen_server:call(?SERVER, {register_mod, Type, Mod, Opts, Seq}). + +%% @doc Unregister an Auth/ACL module. +-spec(unregister_mod(auth | acl, module()) -> ok | {error, not_found | term()}). +unregister_mod(Type, Mod) when Type =:= auth; Type =:= acl -> + gen_server:call(?SERVER, {unregister_mod, Type, Mod}). + +%% @doc Lookup all Auth/ACL modules. +-spec(lookup_mods(auth | acl) -> list()). +lookup_mods(Type) -> + case ets:lookup(?TAB, tab_key(Type)) of + [] -> []; + [{_, Mods}] -> Mods + end. + +tab_key(auth) -> auth_modules; +tab_key(acl) -> acl_modules. + +stop() -> + gen_server:stop(?SERVER, normal, infinity). + +%%----------------------------------------------------------------------------- +%% gen_server callbacks +%%----------------------------------------------------------------------------- + +init([]) -> + ok = emqx_tables:new(?TAB, [set, protected, {read_concurrency, true}]), + {ok, #{}}. + +handle_call({register_mod, Type, Mod, Opts, Seq}, _From, State) -> + Mods = lookup_mods(Type), + reply(case lists:keymember(Mod, 1, Mods) of + true -> {error, already_exists}; + false -> + try Mod:init(Opts) of + {ok, ModState} -> + NewMods = lists:sort(fun({_, _, Seq1}, {_, _, Seq2}) -> + Seq1 >= Seq2 + end, [{Mod, ModState, Seq} | Mods]), + ets:insert(?TAB, {tab_key(Type), NewMods}), + ok + catch + _:Error -> + emqx_logger:error("[AccessControl] Failed to init ~s: ~p", [Mod, Error]), + {error, Error} + end + end, State); + +handle_call({unregister_mod, Type, Mod}, _From, State) -> + Mods = lookup_mods(Type), + reply(case lists:keyfind(Mod, 1, Mods) of + false -> + {error, not_found}; + {Mod, _ModState, _Seq} -> + ets:insert(?TAB, {tab_key(Type), lists:keydelete(Mod, 1, Mods)}), ok + end, State); + +handle_call(stop, _From, State) -> + {stop, normal, ok, State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[AccessControl] unexpected request: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[AccessControl] unexpected msg: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + emqx_logger:error("[AccessControl] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +reply(Reply, State) -> + {reply, Reply, State}. + diff --git a/src/emqx_access_rule.erl b/src/emqx_access_rule.erl new file mode 100644 index 000000000..72a3d09b2 --- /dev/null +++ b/src/emqx_access_rule.erl @@ -0,0 +1,151 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_access_rule). + +-include("emqx.hrl"). + +-type(who() :: all | binary() | + {client, binary()} | + {user, binary()} | + {ipaddr, esockd_cidr:cidr_string()}). + +-type(access() :: subscribe | publish | pubsub). + +-type(rule() :: {allow, all} | + {allow, who(), access(), list(emqx_topic:topic())} | + {deny, all} | + {deny, who(), access(), list(emqx_topic:topic())}). + +-export_type([rule/0]). + +-export([compile/1]). +-export([match/3]). + +-define(ALLOW_DENY(A), ((A =:= allow) orelse (A =:= deny))). +-define(PUBSUB(A), ((A =:= subscribe) orelse (A =:= publish) orelse (A =:= pubsub))). + +%% @doc Compile Access Rule. +compile({A, all}) when ?ALLOW_DENY(A) -> + {A, all}; + +compile({A, Who, Access, Topic}) when ?ALLOW_DENY(A), ?PUBSUB(Access), is_binary(Topic) -> + {A, compile(who, Who), Access, [compile(topic, Topic)]}; + +compile({A, Who, Access, TopicFilters}) when ?ALLOW_DENY(A), ?PUBSUB(Access) -> + {A, compile(who, Who), Access, [compile(topic, Topic) || Topic <- TopicFilters]}. + +compile(who, all) -> + all; +compile(who, {ipaddr, CIDR}) -> + {ipaddr, esockd_cidr:parse(CIDR, true)}; +compile(who, {client, all}) -> + {client, all}; +compile(who, {client, ClientId}) -> + {client, bin(ClientId)}; +compile(who, {user, all}) -> + {user, all}; +compile(who, {user, Username}) -> + {user, bin(Username)}; +compile(who, {'and', Conds}) when is_list(Conds) -> + {'and', [compile(who, Cond) || Cond <- Conds]}; +compile(who, {'or', Conds}) when is_list(Conds) -> + {'or', [compile(who, Cond) || Cond <- Conds]}; + +compile(topic, {eq, Topic}) -> + {eq, emqx_topic:words(bin(Topic))}; +compile(topic, Topic) -> + Words = emqx_topic:words(bin(Topic)), + case 'pattern?'(Words) of + true -> {pattern, Words}; + false -> Words + end. + +'pattern?'(Words) -> + lists:member(<<"%u">>, Words) + orelse lists:member(<<"%c">>, Words). + +bin(L) when is_list(L) -> + list_to_binary(L); +bin(B) when is_binary(B) -> + B. + +%% @doc Match access rule +-spec(match(emqx_types:credentials(), emqx_types:topic(), rule()) + -> {matched, allow} | {matched, deny} | nomatch). +match(_Credentials, _Topic, {AllowDeny, all}) when ?ALLOW_DENY(AllowDeny) -> + {matched, AllowDeny}; +match(Credentials, Topic, {AllowDeny, Who, _PubSub, TopicFilters}) + when ?ALLOW_DENY(AllowDeny) -> + case match_who(Credentials, Who) + andalso match_topics(Credentials, Topic, TopicFilters) of + true -> {matched, AllowDeny}; + false -> nomatch + end. + +match_who(_Credentials, all) -> + true; +match_who(_Credentials, {user, all}) -> + true; +match_who(_Credentials, {client, all}) -> + true; +match_who(#{client_id := ClientId}, {client, ClientId}) -> + true; +match_who(#{username := Username}, {user, Username}) -> + true; +match_who(#{peername := undefined}, {ipaddr, _Tup}) -> + false; +match_who(#{peername := {IP, _}}, {ipaddr, CIDR}) -> + esockd_cidr:match(IP, CIDR); +match_who(Credentials, {'and', Conds}) when is_list(Conds) -> + lists:foldl(fun(Who, Allow) -> + match_who(Credentials, Who) andalso Allow + end, true, Conds); +match_who(Credentials, {'or', Conds}) when is_list(Conds) -> + lists:foldl(fun(Who, Allow) -> + match_who(Credentials, Who) orelse Allow + end, false, Conds); +match_who(_Credentials, _Who) -> + false. + +match_topics(_Credentials, _Topic, []) -> + false; +match_topics(Credentials, Topic, [{pattern, PatternFilter}|Filters]) -> + TopicFilter = feed_var(Credentials, PatternFilter), + match_topic(emqx_topic:words(Topic), TopicFilter) + orelse match_topics(Credentials, Topic, Filters); +match_topics(Credentials, Topic, [TopicFilter|Filters]) -> + match_topic(emqx_topic:words(Topic), TopicFilter) + orelse match_topics(Credentials, Topic, Filters). + +match_topic(Topic, {eq, TopicFilter}) -> + Topic == TopicFilter; +match_topic(Topic, TopicFilter) -> + emqx_topic:match(Topic, TopicFilter). + +feed_var(Credentials, Pattern) -> + feed_var(Credentials, Pattern, []). +feed_var(_Credentials, [], Acc) -> + lists:reverse(Acc); +feed_var(Credentials = #{client_id := undefined}, [<<"%c">>|Words], Acc) -> + feed_var(Credentials, Words, [<<"%c">>|Acc]); +feed_var(Credentials = #{client_id := ClientId}, [<<"%c">>|Words], Acc) -> + feed_var(Credentials, Words, [ClientId |Acc]); +feed_var(Credentials = #{username := undefined}, [<<"%u">>|Words], Acc) -> + feed_var(Credentials, Words, [<<"%u">>|Acc]); +feed_var(Credentials = #{username := Username}, [<<"%u">>|Words], Acc) -> + feed_var(Credentials, Words, [Username|Acc]); +feed_var(Credentials, [W|Words], Acc) -> + feed_var(Credentials, Words, [W|Acc]). + diff --git a/src/emqx_acl_cache.erl b/src/emqx_acl_cache.erl new file mode 100644 index 000000000..5be92814d --- /dev/null +++ b/src/emqx_acl_cache.erl @@ -0,0 +1,221 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_acl_cache). + +-include("emqx.hrl"). + +-export([ get_acl_cache/2 + , put_acl_cache/3 + , cleanup_acl_cache/0 + , empty_acl_cache/0 + , dump_acl_cache/0 + , get_cache_size/0 + , get_cache_max_size/0 + , get_newest_key/0 + , get_oldest_key/0 + , cache_k/2 + , cache_v/1 + , is_enabled/0 + ]). + +-type(acl_result() :: allow | deny). + +%% Wrappers for key and value +cache_k(PubSub, Topic)-> {PubSub, Topic}. +cache_v(AclResult)-> {AclResult, time_now()}. + +-spec(is_enabled() -> boolean()). +is_enabled() -> + application:get_env(emqx, enable_acl_cache, true). + +%% We'll cleanup the cache before repalcing an expired acl. +-spec(get_acl_cache(publish | subscribe, emqx_topic:topic()) -> (acl_result() | not_found)). +get_acl_cache(PubSub, Topic) -> + case erlang:get(cache_k(PubSub, Topic)) of + undefined -> not_found; + {AclResult, CachedAt} -> + if_expired(CachedAt, + fun(false) -> + AclResult; + (true) -> + cleanup_acl_cache(), + not_found + end) + end. + +%% If the cache get full, and also the latest one +%% is expired, then delete all the cache entries +-spec(put_acl_cache(publish | subscribe, emqx_topic:topic(), acl_result()) -> ok). +put_acl_cache(PubSub, Topic, AclResult) -> + MaxSize = get_cache_max_size(), true = (MaxSize =/= 0), + Size = get_cache_size(), + if + Size < MaxSize -> + add_acl(PubSub, Topic, AclResult); + Size =:= MaxSize -> + NewestK = get_newest_key(), + {_AclResult, CachedAt} = erlang:get(NewestK), + if_expired(CachedAt, + fun(true) -> + % all cache expired, cleanup first + empty_acl_cache(), + add_acl(PubSub, Topic, AclResult); + (false) -> + % cache full, perform cache replacement + evict_acl_cache(), + add_acl(PubSub, Topic, AclResult) + end) + end. + +%% delete all the acl entries +-spec(empty_acl_cache() -> ok). +empty_acl_cache() -> + map_acl_cache(fun({CacheK, _CacheV}) -> + erlang:erase(CacheK) + end), + set_cache_size(0), + keys_queue_set(queue:new()). + +%% delete the oldest acl entry +-spec(evict_acl_cache() -> ok). +evict_acl_cache() -> + OldestK = keys_queue_out(), + erlang:erase(OldestK), + decr_cache_size(). + +%% cleanup all the exipired cache entries +-spec(cleanup_acl_cache() -> ok). +cleanup_acl_cache() -> + keys_queue_set( + cleanup_acl(keys_queue_get())). + +get_oldest_key() -> + keys_queue_pick(queue_front()). +get_newest_key() -> + keys_queue_pick(queue_rear()). + +get_cache_max_size() -> + application:get_env(emqx, acl_cache_max_size, 32). + +get_cache_size() -> + case erlang:get(acl_cache_size) of + undefined -> 0; + Size -> Size + end. + +dump_acl_cache() -> + map_acl_cache(fun(Cache) -> Cache end). +map_acl_cache(Fun) -> + [Fun(R) || R = {{SubPub, _T}, _Acl} <- get(), SubPub =:= publish + orelse SubPub =:= subscribe]. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +add_acl(PubSub, Topic, AclResult) -> + K = cache_k(PubSub, Topic), + V = cache_v(AclResult), + case erlang:get(K) of + undefined -> add_new_acl(K, V); + {_AclResult, _CachedAt} -> + update_acl(K, V) + end. + +add_new_acl(K, V) -> + erlang:put(K, V), + keys_queue_in(K), + incr_cache_size(). + +update_acl(K, V) -> + erlang:put(K, V), + keys_queue_update(K). + +cleanup_acl(KeysQ) -> + case queue:out(KeysQ) of + {{value, OldestK}, KeysQ2} -> + {_AclResult, CachedAt} = erlang:get(OldestK), + if_expired(CachedAt, + fun(false) -> KeysQ; + (true) -> + erlang:erase(OldestK), + decr_cache_size(), + cleanup_acl(KeysQ2) + end); + {empty, KeysQ} -> KeysQ + end. + +incr_cache_size() -> + erlang:put(acl_cache_size, get_cache_size() + 1), ok. +decr_cache_size() -> + Size = get_cache_size(), + if Size > 1 -> + erlang:put(acl_cache_size, Size-1); + Size =< 1 -> + erlang:put(acl_cache_size, 0) + end, ok. +set_cache_size(N) -> + erlang:put(acl_cache_size, N), ok. + +%%% Ordered Keys Q %%% +keys_queue_in(Key) -> + %% delete the key first if exists + KeysQ = keys_queue_get(), + keys_queue_set(queue:in(Key, KeysQ)). + +keys_queue_out() -> + case queue:out(keys_queue_get()) of + {{value, OldestK}, Q2} -> + keys_queue_set(Q2), OldestK; + {empty, _Q} -> + undefined + end. + +keys_queue_update(Key) -> + NewKeysQ = keys_queue_remove(Key, keys_queue_get()), + keys_queue_set(queue:in(Key, NewKeysQ)). + +keys_queue_pick(Pick) -> + KeysQ = keys_queue_get(), + case queue:is_empty(KeysQ) of + true -> undefined; + false -> Pick(KeysQ) + end. + +keys_queue_remove(Key, KeysQ) -> + queue:filter(fun + (K) when K =:= Key -> false; (_) -> true + end, KeysQ). + +keys_queue_set(KeysQ) -> + erlang:put(acl_keys_q, KeysQ), ok. +keys_queue_get() -> + case erlang:get(acl_keys_q) of + undefined -> queue:new(); + KeysQ -> KeysQ + end. + +queue_front() -> fun queue:get/1. +queue_rear() -> fun queue:get_r/1. + +time_now() -> erlang:system_time(millisecond). + +if_expired(CachedAt, Fun) -> + TTL = application:get_env(emqx, acl_cache_ttl, 60000), + Now = time_now(), + if (CachedAt + TTL) =< Now -> + Fun(true); + true -> + Fun(false) + end. diff --git a/src/emqx_acl_internal.erl b/src/emqx_acl_internal.erl new file mode 100644 index 000000000..1effd1709 --- /dev/null +++ b/src/emqx_acl_internal.erl @@ -0,0 +1,121 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_acl_internal). + +-behaviour(emqx_acl_mod). + +-include("emqx.hrl"). + +-export([all_rules/0]). + +%% ACL mod callbacks +-export([init/1, check_acl/2, reload_acl/1, description/0]). + +-define(ACL_RULE_TAB, emqx_acl_rule). + +-type(state() :: #{acl_file := string()}). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% @doc Read all rules +-spec(all_rules() -> list(emqx_access_rule:rule())). +all_rules() -> + case ets:lookup(?ACL_RULE_TAB, all_rules) of + [] -> []; + [{_, Rules}] -> Rules + end. + +%%------------------------------------------------------------------------------ +%% ACL callbacks +%%------------------------------------------------------------------------------ + +-spec(init([File :: string()]) -> {ok, #{}}). +init([File]) -> + _ = emqx_tables:new(?ACL_RULE_TAB, [set, public, {read_concurrency, true}]), + ok = load_rules_from_file(File), + {ok, #{acl_file => File}}. + +load_rules_from_file(AclFile) -> + case file:consult(AclFile) of + {ok, Terms} -> + Rules = [emqx_access_rule:compile(Term) || Term <- Terms], + lists:foreach(fun(PubSub) -> + ets:insert(?ACL_RULE_TAB, {PubSub, + lists:filter(fun(Rule) -> filter(PubSub, Rule) end, Rules)}) + end, [publish, subscribe]), + ets:insert(?ACL_RULE_TAB, {all_rules, Terms}), + ok; + {error, Reason} -> + emqx_logger:error("[ACL_INTERNAL] Failed to read ~s: ~p", [AclFile, Reason]), + {error, Reason} + end. + +filter(_PubSub, {allow, all}) -> + true; +filter(_PubSub, {deny, all}) -> + true; +filter(publish, {_AllowDeny, _Who, publish, _Topics}) -> + true; +filter(_PubSub, {_AllowDeny, _Who, pubsub, _Topics}) -> + true; +filter(subscribe, {_AllowDeny, _Who, subscribe, _Topics}) -> + true; +filter(_PubSub, {_AllowDeny, _Who, _, _Topics}) -> + false. + +%% @doc Check ACL +-spec(check_acl({emqx_types:credentials(), emqx_types:pubsub(), emqx_topic:topic()}, #{}) + -> allow | deny | ignore). +check_acl({Credentials, PubSub, Topic}, _State) -> + case match(Credentials, Topic, lookup(PubSub)) of + {matched, allow} -> allow; + {matched, deny} -> deny; + nomatch -> ignore + end. + +lookup(PubSub) -> + case ets:lookup(?ACL_RULE_TAB, PubSub) of + [] -> []; + [{PubSub, Rules}] -> Rules + end. + +match(_Credentials, _Topic, []) -> + nomatch; +match(Credentials, Topic, [Rule|Rules]) -> + case emqx_access_rule:match(Credentials, Topic, Rule) of + nomatch -> + match(Credentials, Topic, Rules); + {matched, AllowDeny} -> + {matched, AllowDeny} + end. + +-spec(reload_acl(state()) -> ok | {error, term()}). +reload_acl(#{acl_file := AclFile}) -> + case catch load_rules_from_file(AclFile) of + ok -> + emqx_logger:info("Reload acl_file ~s successfully", [AclFile]), + ok; + {error, Error} -> + {error, Error}; + {'EXIT', Error} -> + {error, Error} + end. + +-spec(description() -> string()). +description() -> + "Internal ACL with etc/acl.conf". + diff --git a/src/emqttd_acl_mod.erl b/src/emqx_acl_mod.erl similarity index 62% rename from src/emqttd_acl_mod.erl rename to src/emqx_acl_mod.erl index 66e5f098b..716b27967 100644 --- a/src/emqttd_acl_mod.erl +++ b/src/emqx_acl_mod.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,13 +11,10 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_acl_mod). +-module(emqx_acl_mod). --author("Feng Lee "). - --include("emqttd.hrl"). +-include("emqx.hrl"). %%-------------------------------------------------------------------- %% ACL behavihour @@ -26,13 +22,12 @@ -ifdef(use_specs). --callback(init(AclOpts :: list()) -> {ok, State :: any()}). +-callback(init(AclOpts :: list()) -> {ok, State :: term()}). --callback(check_acl({Client :: mqtt_client(), - PubSub :: pubsub(), - Topic :: binary()}, State :: any()) -> allow | deny | ignore). +-callback(check_acl({credentials(), pubsub(), topic()}, State :: term()) + -> allow | deny | ignore). --callback(reload_acl(State :: any()) -> ok | {error, term()}). +-callback(reload_acl(State :: term()) -> ok | {error, term()}). -callback(description() -> string()). diff --git a/src/emqx_alarm_mgr.erl b/src/emqx_alarm_mgr.erl new file mode 100644 index 000000000..fd2a42aa7 --- /dev/null +++ b/src/emqx_alarm_mgr.erl @@ -0,0 +1,143 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_alarm_mgr). + +-behaviour(gen_event). + +-include("emqx.hrl"). + +-export([start_link/0]). +-export([alarm_fun/0, get_alarms/0, set_alarm/1, clear_alarm/1]). +-export([add_alarm_handler/1, add_alarm_handler/2, delete_alarm_handler/1]). + +%% gen_event callbacks +-export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2, + code_change/3]). + +-define(ALARM_MGR, ?MODULE). + +start_link() -> + start_with( + fun(Pid) -> + gen_event:add_handler(Pid, ?MODULE, []) + end). + +start_with(Fun) -> + case gen_event:start_link({local, ?ALARM_MGR}) of + {ok, Pid} -> Fun(Pid), {ok, Pid}; + Error -> Error + end. + +alarm_fun() -> alarm_fun(false). + +alarm_fun(Bool) -> + fun(alert, _Alarm) when Bool =:= true -> alarm_fun(true); + (alert, Alarm) when Bool =:= false -> set_alarm(Alarm), alarm_fun(true); + (clear, AlarmId) when Bool =:= true -> clear_alarm(AlarmId), alarm_fun(false); + (clear, _AlarmId) when Bool =:= false -> alarm_fun(false) + end. + +-spec(set_alarm(emqx_types:alarm()) -> ok). +set_alarm(Alarm) when is_record(Alarm, alarm) -> + gen_event:notify(?ALARM_MGR, {set_alarm, Alarm}). + +-spec(clear_alarm(any()) -> ok). +clear_alarm(AlarmId) when is_binary(AlarmId) -> + gen_event:notify(?ALARM_MGR, {clear_alarm, AlarmId}). + +-spec(get_alarms() -> list(emqx_types:alarm())). +get_alarms() -> + gen_event:call(?ALARM_MGR, ?MODULE, get_alarms). + +add_alarm_handler(Module) when is_atom(Module) -> + gen_event:add_handler(?ALARM_MGR, Module, []). + +add_alarm_handler(Module, Args) when is_atom(Module) -> + gen_event:add_handler(?ALARM_MGR, Module, Args). + +delete_alarm_handler(Module) when is_atom(Module) -> + gen_event:delete_handler(?ALARM_MGR, Module, []). + +%%------------------------------------------------------------------------------ +%% Default Alarm handler +%%------------------------------------------------------------------------------ + +init(_) -> {ok, #{alarms => []}}. + +handle_event({set_alarm, Alarm = #alarm{timestamp = undefined}}, State)-> + handle_event({set_alarm, Alarm#alarm{timestamp = os:timestamp()}}, State); + +handle_event({set_alarm, Alarm = #alarm{id = AlarmId}}, State = #{alarms := Alarms}) -> + case encode_alarm(Alarm) of + {ok, Json} -> + emqx_broker:safe_publish(alarm_msg(alert, AlarmId, Json)); + {error, Reason} -> + emqx_logger:error("[AlarmMgr] Failed to encode alarm: ~p", [Reason]) + end, + {ok, State#{alarms := [Alarm|Alarms]}}; + +handle_event({clear_alarm, AlarmId}, State = #{alarms := Alarms}) -> + case emqx_json:safe_encode([{id, AlarmId}, {ts, os:system_time(second)}]) of + {ok, Json} -> + emqx_broker:safe_publish(alarm_msg(clear, AlarmId, Json)); + {error, Reason} -> + emqx_logger:error("[AlarmMgr] Failed to encode clear: ~p", [Reason]) + end, + {ok, State#{alarms := lists:keydelete(AlarmId, 2, Alarms)}, hibernate}; + +handle_event(Event, State)-> + emqx_logger:error("[AlarmMgr] unexpected event: ~p", [Event]), + {ok, State}. + +handle_info(Info, State) -> + emqx_logger:error("[AlarmMgr] unexpected info: ~p", [Info]), + {ok, State}. + +handle_call(get_alarms, State = #{alarms := Alarms}) -> + {ok, Alarms, State}; + +handle_call(Req, State) -> + emqx_logger:error("[AlarmMgr] unexpected call: ~p", [Req]), + {ok, ignored, State}. + +terminate(swap, State) -> + {?MODULE, State}; +terminate(_, _) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +encode_alarm(#alarm{id = AlarmId, severity = Severity, title = Title, + summary = Summary, timestamp = Ts}) -> + emqx_json:safe_encode([{id, AlarmId}, {severity, Severity}, + {title, iolist_to_binary(Title)}, + {summary, iolist_to_binary(Summary)}, + {ts, emqx_time:now_secs(Ts)}]). + +alarm_msg(Type, AlarmId, Json) -> + Msg = emqx_message:make(?ALARM_MGR, topic(Type, AlarmId), Json), + emqx_message:set_headers( #{'Content-Type' => <<"application/json">>}, + emqx_message:set_flag(sys, Msg)). + +topic(alert, AlarmId) -> + emqx_topic:systop(<<"alarms/", AlarmId/binary, "/alert">>); +topic(clear, AlarmId) -> + emqx_topic:systop(<<"alarms/", AlarmId/binary, "/clear">>). + diff --git a/src/emqx_app.erl b/src/emqx_app.erl new file mode 100644 index 000000000..977eabc2f --- /dev/null +++ b/src/emqx_app.erl @@ -0,0 +1,71 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +-define(APP, emqx). + +%%-------------------------------------------------------------------- +%% Application callbacks +%%-------------------------------------------------------------------- + +start(_Type, _Args) -> + %% We'd like to configure the primary logger level here, rather than set the + %% kernel config `logger_level` before starting the erlang vm. + %% This is because the latter approach an annoying debug msg will be printed out: + %% "[debug] got_unexpected_message {'EXIT',<0.1198.0>,normal}" + logger:set_primary_config(level, application:get_env(emqx, primary_log_level, error)), + + print_banner(), + ekka:start(), + {ok, Sup} = emqx_sup:start_link(), + emqx_modules:load(), + emqx_plugins:init(), + emqx_plugins:load(), + emqx_listeners:start(), + start_autocluster(), + register(emqx, self()), + print_vsn(), + {ok, Sup}. + +-spec(stop(State :: term()) -> term()). +stop(_State) -> + emqx_listeners:stop(), + emqx_modules:unload(). + +%%-------------------------------------------------------------------- +%% Print Banner +%%-------------------------------------------------------------------- + +print_banner() -> + io:format("Starting ~s on node ~s~n", [?APP, node()]). + +print_vsn() -> + {ok, Descr} = application:get_key(description), + {ok, Vsn} = application:get_key(vsn), + io:format("~s ~s is running now!~n", [Descr, Vsn]). + +%%-------------------------------------------------------------------- +%% Autocluster +%%-------------------------------------------------------------------- + +start_autocluster() -> + ekka:callback(prepare, fun emqx:shutdown/1), + ekka:callback(reboot, fun emqx:reboot/0), + ekka:autocluster(?APP). + diff --git a/test/emqttd_cli_SUITE.erl b/src/emqx_auth_mod.erl similarity index 54% rename from test/emqttd_cli_SUITE.erl rename to src/emqx_auth_mod.erl index 024432d95..00ecb659a 100644 --- a/test/emqttd_cli_SUITE.erl +++ b/src/emqx_auth_mod.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,41 +11,31 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. + +-module(emqx_auth_mod). + +-include("emqx.hrl"). + +%%-------------------------------------------------------------------- +%% Authentication behavihour %%-------------------------------------------------------------------- --module(emqttd_cli_SUITE). +-ifdef(use_specs). --compile(export_all). +-callback(init(AuthOpts :: list()) -> {ok, State :: term()}). --include("emqttd.hrl"). +-callback(check(credentials(), password(), State :: term()) + -> ok | {ok, boolean()} | {ok, map()} | + {continue, map()} | ignore | {error, term()}). +-callback(description() -> string()). --include_lib("eunit/include/eunit.hrl"). +-else. -all() -> - [{group, subscriptions}]. +-export([behaviour_info/1]). -groups() -> - [{subscriptions, [sequence], - [t_subsciptions_list, - t_subsciptions_show, - t_subsciptions_add, - t_subsciptions_del]}]. - -init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - todo. - -t_subsciptions_list(_) -> - todo. - -t_subsciptions_show(_) -> - todo. - -t_subsciptions_add(_) -> - todo. - -t_subsciptions_del(_) -> - todo. +behaviour_info(callbacks) -> + [{init, 1}, {check, 3}, {description, 0}]; +behaviour_info(_Other) -> + undefined. +-endif. diff --git a/src/emqx_banned.erl b/src/emqx_banned.erl new file mode 100644 index 000000000..0c9ffdd10 --- /dev/null +++ b/src/emqx_banned.erl @@ -0,0 +1,119 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_banned). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-export([start_link/0]). +-export([check/1]). +-export([add/1, delete/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(TAB, ?MODULE). + +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?TAB, [ + {type, set}, + {disc_copies, [node()]}, + {record_name, banned}, + {attributes, record_info(fields, banned)}, + {storage_properties, [{ets, [{read_concurrency, true}]}]}]); + +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?TAB). + +%% @doc Start the banned server. +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec(check(emqx_types:credentials()) -> boolean()). +check(#{client_id := ClientId, username := Username, peername := {IPAddr, _}}) -> + ets:member(?TAB, {client_id, ClientId}) + orelse ets:member(?TAB, {username, Username}) + orelse ets:member(?TAB, {ipaddr, IPAddr}). + +-spec(add(#banned{}) -> ok). +add(Banned) when is_record(Banned, banned) -> + mnesia:dirty_write(?TAB, Banned). + +-spec(delete({client_id, emqx_types:client_id()} + | {username, emqx_types:username()} + | {peername, emqx_types:peername()}) -> ok). +delete(Key) -> + mnesia:dirty_delete(?TAB, Key). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + {ok, ensure_expiry_timer(#{expiry_timer => undefined})}. + +handle_call(Req, _From, State) -> + emqx_logger:error("[Banned] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[Banned] unexpected msg: ~p", [Msg]), + {noreply, State}. + +handle_info({timeout, TRef, expire}, State = #{expiry_timer := TRef}) -> + mnesia:async_dirty(fun expire_banned_items/1, [erlang:system_time(second)]), + {noreply, ensure_expiry_timer(State), hibernate}; + +handle_info(Info, State) -> + emqx_logger:error("[Banned] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{expiry_timer := TRef}) -> + emqx_misc:cancel_timer(TRef). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +-ifdef(TEST). +ensure_expiry_timer(State) -> + State#{expiry_timer := emqx_misc:start_timer(timer:seconds(1), expire)}. +-else. +ensure_expiry_timer(State) -> + State#{expiry_timer := emqx_misc:start_timer(timer:minutes(1), expire)}. +-endif. + +expire_banned_items(Now) -> + mnesia:foldl( + fun(B = #banned{until = Until}, _Acc) when Until < Now -> + mnesia:delete_object(?TAB, B, sticky_write); + (_, _Acc) -> ok + end, ok, ?TAB). + diff --git a/src/emqx_base62.erl b/src/emqx_base62.erl new file mode 100644 index 000000000..1a9db245a --- /dev/null +++ b/src/emqx_base62.erl @@ -0,0 +1,112 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_base62). + +-export([encode/1, + encode/2, + decode/1, + decode/2]). + +%% @doc Encode any data to base62 binary +-spec encode(string() + | integer() + | binary()) -> binary(). +encode(I) when is_integer(I) -> + encode(integer_to_binary(I)); +encode(S) when is_list(S)-> + encode(list_to_binary(S)); +encode(B) when is_binary(B) -> + encode(B, <<>>). + +%% encode(D, string) -> +%% binary_to_list(encode(D)). + +%% @doc Decode base62 binary to origin data binary +decode(L) when is_list(L) -> + decode(list_to_binary(L)); +decode(B) when is_binary(B) -> + decode(B, <<>>). + + + +%%==================================================================== +%% Internal functions +%%==================================================================== + +encode(D, string) -> + binary_to_list(encode(D)); +encode(<>, Acc) -> + CharList = [encode_char(Index1), encode_char(Index2), encode_char(Index3), encode_char(Index4)], + NewAcc = <>, + encode(Rest, NewAcc); +encode(<>, Acc) -> + CharList = [encode_char(Index1), encode_char(Index2), encode_char(Index3)], + NewAcc = <>, + encode(<<>>, NewAcc); +encode(<>, Acc) -> + CharList = [encode_char(Index1), encode_char(Index2)], + NewAcc = <>, + encode(<<>>, NewAcc); +encode(<<>>, Acc) -> + Acc. + +decode(D, integer) -> + binary_to_integer(decode(D)); +decode(D, string) -> + binary_to_list(decode(D)); +decode(<>, Acc) + when bit_size(Rest) >= 8-> + case Head == $9 of + true -> + <> = Rest, + DecodeChar = decode_char(9, Head1), + <<_:2, RestBit:6>> = <>, + NewAcc = <>, + decode(Rest1, NewAcc); + false -> + DecodeChar = decode_char(Head), + <<_:2, RestBit:6>> = <>, + NewAcc = <>, + decode(Rest, NewAcc) + end; +decode(<>, Acc) -> + DecodeChar = decode_char(Head), + LeftBitSize = bit_size(Acc) rem 8, + RightBitSize = 8 - LeftBitSize, + <<_:LeftBitSize, RestBit:RightBitSize>> = <>, + NewAcc = <>, + decode(Rest, NewAcc); +decode(<<>>, Acc) -> + Acc. + + +encode_char(I) when I < 26 -> + $A + I; +encode_char(I) when I < 52 -> + $a + I - 26; +encode_char(I) when I < 61 -> + $0 + I - 52; +encode_char(I) -> + [$9, $A + I - 61]. + +decode_char(I) when I >= $a andalso I =< $z -> + I + 26 - $a; +decode_char(I) when I >= $0 andalso I =< $8-> + I + 52 - $0; +decode_char(I) when I >= $A andalso I =< $Z-> + I - $A. + +decode_char(9, I) -> + I + 61 - $A. diff --git a/src/emqx_batch.erl b/src/emqx_batch.erl new file mode 100644 index 000000000..ffa9a7224 --- /dev/null +++ b/src/emqx_batch.erl @@ -0,0 +1,74 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_batch). + +-export([init/1, push/2, commit/1]). +-export([size/1, items/1]). + +-type(options() :: #{ + batch_size => non_neg_integer(), + linger_ms => pos_integer(), + commit_fun := function() + }). +-export_type([options/0]). + +-record(batch, { + batch_size :: non_neg_integer(), + batch_q :: list(any()), + linger_ms :: pos_integer(), + linger_timer :: reference() | undefined, + commit_fun :: function() + }). +-type(batch() :: #batch{}). +-export_type([batch/0]). + +-spec(init(options()) -> batch()). +init(Opts) when is_map(Opts) -> + #batch{batch_size = maps:get(batch_size, Opts, 1000), + batch_q = [], + linger_ms = maps:get(linger_ms, Opts, 1000), + commit_fun = maps:get(commit_fun, Opts)}. + +-spec(push(any(), batch()) -> batch()). +push(El, Batch = #batch{batch_q = Q, linger_ms = Ms, linger_timer = undefined}) when length(Q) == 0 -> + Batch#batch{batch_q = [El], linger_timer = erlang:send_after(Ms, self(), batch_linger_expired)}; + +%% no limit. +push(El, Batch = #batch{batch_size = 0, batch_q = Q}) -> + Batch#batch{batch_q = [El|Q]}; + +push(El, Batch = #batch{batch_size = MaxSize, batch_q = Q}) when length(Q) >= MaxSize -> + commit(Batch#batch{batch_q = [El|Q]}); + +push(El, Batch = #batch{batch_q = Q}) -> + Batch#batch{batch_q = [El|Q]}. + +-spec(commit(batch()) -> batch()). +commit(Batch = #batch{batch_q = Q, commit_fun = Commit}) -> + _ = Commit(lists:reverse(Q)), + reset(Batch). + +reset(Batch = #batch{linger_timer = TRef}) -> + _ = emqx_misc:cancel_timer(TRef), + Batch#batch{batch_q = [], linger_timer = undefined}. + +-spec(size(batch()) -> non_neg_integer()). +size(#batch{batch_q = Q}) -> + length(Q). + +-spec(items(batch()) -> list(any())). +items(#batch{batch_q = Q}) -> + lists:reverse(Q). + diff --git a/src/emqx_bridge.erl b/src/emqx_bridge.erl new file mode 100644 index 000000000..b1ccdc44b --- /dev/null +++ b/src/emqx_bridge.erl @@ -0,0 +1,334 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_bridge). + +-behaviour(gen_server). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-import(proplists, [get_value/2, get_value/3]). + +-export([start_link/2, start_bridge/1, stop_bridge/1, status/1]). + +-export([show_forwards/1, add_forward/2, del_forward/2]). + +-export([show_subscriptions/1, add_subscription/3, del_subscription/2]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-record(state, {client_pid, options, reconnect_interval, + mountpoint, queue, mqueue_type, max_pending_messages, + forwards = [], subscriptions = []}). + +-record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false, + packet_id, topic, props, payload}). + +start_link(Name, Options) -> + gen_server:start_link({local, name(Name)}, ?MODULE, [Options], []). + +start_bridge(Name) -> + gen_server:call(name(Name), start_bridge). + +stop_bridge(Name) -> + gen_server:call(name(Name), stop_bridge). + +-spec(show_forwards(atom()) -> list()). +show_forwards(Name) -> + gen_server:call(name(Name), show_forwards). + +-spec(add_forward(atom(), binary()) -> ok | {error, already_exists | validate_fail}). +add_forward(Name, Topic) -> + case catch emqx_topic:validate({filter, Topic}) of + true -> + gen_server:call(name(Name), {add_forward, Topic}); + {'EXIT', _Reason} -> + {error, validate_fail} + end. + +-spec(del_forward(atom(), binary()) -> ok | {error, validate_fail}). +del_forward(Name, Topic) -> + case catch emqx_topic:validate({filter, Topic}) of + true -> + gen_server:call(name(Name), {del_forward, Topic}); + _ -> + {error, validate_fail} + end. + +-spec(show_subscriptions(atom()) -> list()). +show_subscriptions(Name) -> + gen_server:call(name(Name), show_subscriptions). + +-spec(add_subscription(atom(), binary(), integer()) -> ok | {error, already_exists | validate_fail}). +add_subscription(Name, Topic, QoS) -> + case catch emqx_topic:validate({filter, Topic}) of + true -> + gen_server:call(name(Name), {add_subscription, Topic, QoS}); + {'EXIT', _Reason} -> + {error, validate_fail} + end. + +-spec(del_subscription(atom(), binary()) -> ok | {error, validate_fail}). +del_subscription(Name, Topic) -> + case catch emqx_topic:validate({filter, Topic}) of + true -> + gen_server:call(name(Name), {del_subscription, Topic}); + _ -> + {error, validate_fail} + end. + +status(Pid) -> + gen_server:call(Pid, status). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Options]) -> + process_flag(trap_exit, true), + case get_value(start_type, Options, manual) of + manual -> ok; + auto -> erlang:send_after(1000, self(), start) + end, + ReconnectInterval = get_value(reconnect_interval, Options, 30000), + MaxPendingMsg = get_value(max_pending_messages, Options, 10000), + Mountpoint = format_mountpoint(get_value(mountpoint, Options)), + MqueueType = get_value(mqueue_type, Options, memory), + Queue = [], + {ok, #state{mountpoint = Mountpoint, + queue = Queue, + mqueue_type = MqueueType, + options = Options, + reconnect_interval = ReconnectInterval, + max_pending_messages = MaxPendingMsg}}. + +handle_call(start_bridge, _From, State = #state{client_pid = undefined}) -> + {noreply, NewState} = handle_info(start, State), + {reply, #{msg => <<"start bridge successfully">>}, NewState}; + +handle_call(start_bridge, _From, State) -> + {reply, #{msg => <<"bridge already started">>}, State}; + +handle_call(stop_bridge, _From, State = #state{client_pid = undefined}) -> + {reply, #{msg => <<"bridge not started">>}, State}; + +handle_call(stop_bridge, _From, State = #state{client_pid = Pid}) -> + emqx_client:disconnect(Pid), + {reply, #{msg => <<"stop bridge successfully">>}, State}; + +handle_call(status, _From, State = #state{client_pid = undefined}) -> + {reply, #{status => <<"Stopped">>}, State}; +handle_call(status, _From, State = #state{client_pid = _Pid})-> + {reply, #{status => <<"Running">>}, State}; + +handle_call(show_forwards, _From, State = #state{forwards = Forwards}) -> + {reply, Forwards, State}; + +handle_call({add_forward, Topic}, _From, State = #state{forwards = Forwards}) -> + case not lists:member(Topic, Forwards) of + true -> + emqx_broker:subscribe(Topic), + {reply, ok, State#state{forwards = [Topic | Forwards]}}; + false -> + {reply, {error, already_exists}, State} + end; + +handle_call({del_forward, Topic}, _From, State = #state{forwards = Forwards}) -> + case lists:member(Topic, Forwards) of + true -> + emqx_broker:unsubscribe(Topic), + {reply, ok, State#state{forwards = lists:delete(Topic, Forwards)}}; + false -> + {reply, ok, State} + end; + +handle_call(show_subscriptions, _From, State = #state{subscriptions = Subscriptions}) -> + {reply, Subscriptions, State}; + +handle_call({add_subscription, Topic, Qos}, _From, State = #state{subscriptions = Subscriptions, client_pid = ClientPid}) -> + case not lists:keymember(Topic, 1, Subscriptions) of + true -> + emqx_client:subscribe(ClientPid, {Topic, Qos}), + {reply, ok, State#state{subscriptions = [{Topic, Qos} | Subscriptions]}}; + false -> + {reply, {error, already_exists}, State} + end; + +handle_call({del_subscription, Topic}, _From, State = #state{subscriptions = Subscriptions, client_pid = ClientPid}) -> + case lists:keymember(Topic, 1, Subscriptions) of + true -> + emqx_client:unsubscribe(ClientPid, Topic), + {reply, ok, State#state{subscriptions = lists:keydelete(Topic, 1, Subscriptions)}}; + false -> + {reply, ok, State} + end; + +handle_call(Req, _From, State) -> + emqx_logger:error("[Bridge] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[Bridge] unexpected cast: ~p", [Msg]), + {noreply, State}. + +%%---------------------------------------------------------------- +%% start message bridge +%%---------------------------------------------------------------- +handle_info(start, State = #state{options = Options, + client_pid = undefined}) -> + case emqx_client:start_link([{owner, self()}|options(Options)]) of + {ok, ClientPid} -> + case emqx_client:connect(ClientPid) of + {ok, _} -> + emqx_logger:info("[Bridge] connected to remote sucessfully"), + Subs = subscribe_remote_topics(ClientPid, get_value(subscriptions, Options, [])), + Forwards = subscribe_local_topics(get_value(forwards, Options, [])), + {noreply, State#state{client_pid = ClientPid, + subscriptions = Subs, + forwards = Forwards}}; + {error, Reason} -> + emqx_logger:error("[Bridge] connect to remote failed! error: ~p", [Reason]), + {noreply, State#state{client_pid = ClientPid}} + end; + {error, Reason} -> + emqx_logger:error("[Bridge] start failed! error: ~p", [Reason]), + {noreply, State} + end; + +%%---------------------------------------------------------------- +%% received local node message +%%---------------------------------------------------------------- +handle_info({dispatch, _, #message{topic = Topic, payload = Payload, flags = #{retain := Retain}}}, + State = #state{client_pid = Pid, mountpoint = Mountpoint, queue = Queue, + mqueue_type = MqueueType, max_pending_messages = MaxPendingMsg}) -> + Msg = #mqtt_msg{qos = 1, + retain = Retain, + topic = mountpoint(Mountpoint, Topic), + payload = Payload}, + case emqx_client:publish(Pid, Msg) of + {ok, PkgId} -> + {noreply, State#state{queue = store(MqueueType, {PkgId, Msg}, Queue, MaxPendingMsg)}}; + {error, Reason} -> + emqx_logger:error("[Bridge] Publish fail:~p", [Reason]), + {noreply, State} + end; + +%%---------------------------------------------------------------- +%% received remote node message +%%---------------------------------------------------------------- +handle_info({publish, #{qos := QoS, dup := Dup, retain := Retain, topic := Topic, + properties := Props, payload := Payload}}, State) -> + NewMsg0 = emqx_message:make(bridge, QoS, Topic, Payload), + NewMsg1 = emqx_message:set_headers(Props, emqx_message:set_flags(#{dup => Dup, retain => Retain}, NewMsg0)), + emqx_broker:publish(NewMsg1), + {noreply, State}; + +%%---------------------------------------------------------------- +%% received remote puback message +%%---------------------------------------------------------------- +handle_info({puback, #{packet_id := PkgId}}, State = #state{queue = Queue, mqueue_type = MqueueType}) -> + % lists:keydelete(PkgId, 1, Queue) + {noreply, State#state{queue = delete(MqueueType, PkgId, Queue)}}; + +handle_info({'EXIT', Pid, normal}, State = #state{client_pid = Pid}) -> + emqx_logger:warning("[Bridge] stop ~p", [normal]), + {noreply, State#state{client_pid = undefined}}; + +handle_info({'EXIT', Pid, Reason}, State = #state{client_pid = Pid, + reconnect_interval = ReconnectInterval}) -> + emqx_logger:error("[Bridge] stop ~p", [Reason]), + erlang:send_after(ReconnectInterval, self(), start), + {noreply, State#state{client_pid = undefined}}; + +handle_info(Info, State) -> + emqx_logger:error("[Bridge] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #state{}) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +subscribe_remote_topics(ClientPid, Subscriptions) -> + [begin emqx_client:subscribe(ClientPid, {bin(Topic), Qos}), {bin(Topic), Qos} end + || {Topic, Qos} <- Subscriptions, emqx_topic:validate({filter, bin(Topic)})]. + +subscribe_local_topics(Topics) -> + [begin emqx_broker:subscribe(bin(Topic)), bin(Topic) end + || Topic <- Topics, emqx_topic:validate({filter, bin(Topic)})]. + +proto_ver(mqttv3) -> v3; +proto_ver(mqttv4) -> v4; +proto_ver(mqttv5) -> v5. +address(Address) -> + case string:tokens(Address, ":") of + [Host] -> {Host, 1883}; + [Host, Port] -> {Host, list_to_integer(Port)} + end. +options(Options) -> + options(Options, []). +options([], Acc) -> + Acc; +options([{username, Username}| Options], Acc) -> + options(Options, [{username, Username}|Acc]); +options([{proto_ver, ProtoVer}| Options], Acc) -> + options(Options, [{proto_ver, proto_ver(ProtoVer)}|Acc]); +options([{password, Password}| Options], Acc) -> + options(Options, [{password, Password}|Acc]); +options([{keepalive, Keepalive}| Options], Acc) -> + options(Options, [{keepalive, Keepalive}|Acc]); +options([{client_id, ClientId}| Options], Acc) -> + options(Options, [{client_id, ClientId}|Acc]); +options([{clean_start, CleanStart}| Options], Acc) -> + options(Options, [{clean_start, CleanStart}|Acc]); +options([{address, Address}| Options], Acc) -> + {Host, Port} = address(Address), + options(Options, [{host, Host}, {port, Port}|Acc]); +options([{ssl, Ssl}| Options], Acc) -> + options(Options, [{ssl, Ssl}|Acc]); +options([{ssl_opts, SslOpts}| Options], Acc) -> + options(Options, [{ssl_opts, SslOpts}|Acc]); +options([_Option | Options], Acc) -> + options(Options, Acc). + +name(Id) -> + list_to_atom(lists:concat([?MODULE, "_", Id])). + +bin(L) -> iolist_to_binary(L). + +mountpoint(undefined, Topic) -> + Topic; +mountpoint(Prefix, Topic) -> + <>. + +format_mountpoint(undefined) -> + undefined; +format_mountpoint(Prefix) -> + binary:replace(bin(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). + +store(memory, Data, Queue, MaxPendingMsg) when length(Queue) =< MaxPendingMsg -> + [Data | Queue]; +store(memory, _Data, Queue, _MaxPendingMsg) -> + logger:error("Beyond max pending messages"), + Queue; +store(disk, Data, Queue, _MaxPendingMsg)-> + [Data | Queue]. + +delete(memory, PkgId, Queue) -> + lists:keydelete(PkgId, 1, Queue); +delete(disk, PkgId, Queue) -> + lists:keydelete(PkgId, 1, Queue). diff --git a/src/emqx_bridge_sup.erl b/src/emqx_bridge_sup.erl new file mode 100644 index 000000000..3911da2a6 --- /dev/null +++ b/src/emqx_bridge_sup.erl @@ -0,0 +1,45 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_bridge_sup). + +-behavior(supervisor). + +-include("emqx.hrl"). + +-export([start_link/0, bridges/0]). + +%% Supervisor callbacks +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%% @doc List all bridges +-spec(bridges() -> [{node(), map()}]). +bridges() -> + [{Name, emqx_bridge:status(Pid)} || {Name, Pid, _, _} <- supervisor:which_children(?MODULE)]. + +init([]) -> + BridgesOpts = emqx_config:get_env(bridges, []), + Bridges = [spec(Opts)|| Opts <- BridgesOpts], + {ok, {{one_for_one, 10, 100}, Bridges}}. + +spec({Id, Options})-> + #{id => Id, + start => {emqx_bridge, start_link, [Id, Options]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_bridge]}. diff --git a/src/emqx_broker.erl b/src/emqx_broker.erl new file mode 100644 index 000000000..429c6097d --- /dev/null +++ b/src/emqx_broker.erl @@ -0,0 +1,445 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_broker). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/2]). +-export([subscribe/1, subscribe/2, subscribe/3]). +-export([unsubscribe/1]). +-export([subscriber_down/1]). +-export([publish/1, safe_publish/1]). +-export([dispatch/2]). +-export([subscriptions/1, subscribers/1, subscribed/2]). +-export([get_subopts/2, set_subopts/2]). +-export([topics/0]). + +%% Stats fun +-export([stats_fun/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-import(emqx_tables, [lookup_value/2, lookup_value/3]). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +-define(BROKER, ?MODULE). + +%% ETS tables for PubSub +-define(SUBOPTION, emqx_suboption). +-define(SUBSCRIBER, emqx_subscriber). +-define(SUBSCRIPTION, emqx_subscription). + +%% Guards +-define(is_subid(Id), (is_binary(Id) orelse is_atom(Id))). + +-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). +start_link(Pool, Id) -> + ok = create_tabs(), + gen_server:start_link({local, emqx_misc:proc_name(?BROKER, Id)}, + ?MODULE, [Pool, Id], []). + +%%------------------------------------------------------------------------------ +%% Create tabs +%%------------------------------------------------------------------------------ + +-spec(create_tabs() -> ok). +create_tabs() -> + TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}], + + %% SubOption: {SubPid, Topic} -> SubOption + ok = emqx_tables:new(?SUBOPTION, [set | TabOpts]), + + %% Subscription: SubPid -> Topic1, Topic2, Topic3, ... + %% duplicate_bag: o(1) insert + ok = emqx_tables:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]), + + %% Subscriber: Topic -> SubPid1, SubPid2, SubPid3, ... + %% bag: o(n) insert:( + ok = emqx_tables:new(?SUBSCRIBER, [bag | TabOpts]). + +%%------------------------------------------------------------------------------ +%% Subscribe API +%%------------------------------------------------------------------------------ + +-spec(subscribe(emqx_topic:topic()) -> ok). +subscribe(Topic) when is_binary(Topic) -> + subscribe(Topic, undefined). + +-spec(subscribe(emqx_topic:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok). +subscribe(Topic, SubId) when is_binary(Topic), ?is_subid(SubId) -> + subscribe(Topic, SubId, #{qos => 0}); +subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> + subscribe(Topic, undefined, SubOpts). + +-spec(subscribe(emqx_topic:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok). +subscribe(Topic, SubId, SubOpts) when is_binary(Topic), ?is_subid(SubId), is_map(SubOpts) -> + SubPid = self(), + case ets:member(?SUBOPTION, {SubPid, Topic}) of + false -> + ok = emqx_broker_helper:register_sub(SubPid, SubId), + do_subscribe(Topic, SubPid, with_subid(SubId, SubOpts)); + true -> ok + end. + +with_subid(undefined, SubOpts) -> + SubOpts; +with_subid(SubId, SubOpts) -> + maps:put(subid, SubId, SubOpts). + +%% @private +do_subscribe(Topic, SubPid, SubOpts) -> + true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), + Group = maps:get(share, SubOpts, undefined), + do_subscribe(Group, Topic, SubPid, SubOpts). + +do_subscribe(undefined, Topic, SubPid, SubOpts) -> + case emqx_broker_helper:get_sub_shard(SubPid, Topic) of + 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + call(pick(Topic), {subscribe, Topic}); + I -> true = ets:insert(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, maps:put(shard, I, SubOpts)}), + call(pick({Topic, I}), {subscribe, Topic, I}) + end; + +%% Shared subscription +do_subscribe(Group, Topic, SubPid, SubOpts) -> + true = ets:insert(?SUBOPTION, {{SubPid, Topic}, SubOpts}), + emqx_shared_sub:subscribe(Group, Topic, SubPid). + +%%------------------------------------------------------------------------------ +%% Unsubscribe API +%%------------------------------------------------------------------------------ + +-spec(unsubscribe(emqx_topic:topic()) -> ok). +unsubscribe(Topic) when is_binary(Topic) -> + SubPid = self(), + case ets:lookup(?SUBOPTION, {SubPid, Topic}) of + [{_, SubOpts}] -> + _ = emqx_broker_helper:reclaim_seq(Topic), + do_unsubscribe(Topic, SubPid, SubOpts); + [] -> ok + end. + +do_unsubscribe(Topic, SubPid, SubOpts) -> + true = ets:delete(?SUBOPTION, {SubPid, Topic}), + true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), + Group = maps:get(share, SubOpts, undefined), + do_unsubscribe(Group, Topic, SubPid, SubOpts). + +do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; + +do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> + emqx_shared_sub:unsubscribe(Group, Topic, SubPid). + +%%------------------------------------------------------------------------------ +%% Publish +%%------------------------------------------------------------------------------ + +-spec(publish(emqx_types:message()) -> emqx_types:deliver_results()). +publish(Msg) when is_record(Msg, message) -> + _ = emqx_tracer:trace(publish, Msg), + case emqx_hooks:run('message.publish', [], Msg) of + {ok, Msg1 = #message{topic = Topic}} -> + Delivery = route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1)), + Delivery#delivery.results; + {stop, _} -> + emqx_logger:warning("Stop publishing: ~s", [emqx_message:format(Msg)]), + [] + end. + +%% Called internally +-spec(safe_publish(emqx_types:message()) -> ok). +safe_publish(Msg) when is_record(Msg, message) -> + try + publish(Msg) + catch + _:Error:Stacktrace -> + emqx_logger:error("[Broker] publish error: ~p~n~p~n~p", [Error, Msg, Stacktrace]) + after + ok + end. + +delivery(Msg) -> + #delivery{sender = self(), message = Msg, results = []}. + +%%------------------------------------------------------------------------------ +%% Route +%%------------------------------------------------------------------------------ + +route([], Delivery = #delivery{message = Msg}) -> + emqx_hooks:run('message.dropped', [#{node => node()}, Msg]), + inc_dropped_cnt(Msg#message.topic), Delivery; + +route([{To, Node}], Delivery) when Node =:= node() -> + dispatch(To, Delivery); + +route([{To, Node}], Delivery = #delivery{results = Results}) when is_atom(Node) -> + forward(Node, To, Delivery#delivery{results = [{route, Node, To}|Results]}); + +route([{To, Group}], Delivery) when is_tuple(Group); is_binary(Group) -> + emqx_shared_sub:dispatch(Group, To, Delivery); + +route(Routes, Delivery) -> + lists:foldl(fun(Route, Acc) -> route([Route], Acc) end, Delivery, Routes). + +aggre([]) -> + []; +aggre([#route{topic = To, dest = Node}]) when is_atom(Node) -> + [{To, Node}]; +aggre([#route{topic = To, dest = {Group, _Node}}]) -> + [{To, Group}]; +aggre(Routes) -> + lists:foldl( + fun(#route{topic = To, dest = Node}, Acc) when is_atom(Node) -> + [{To, Node} | Acc]; + (#route{topic = To, dest = {Group, _Node}}, Acc) -> + lists:usort([{To, Group} | Acc]) + end, [], Routes). + +%% @doc Forward message to another node. +forward(Node, To, Delivery) -> + %% rpc:call to ensure the delivery, but the latency:( + case emqx_rpc:call(Node, ?BROKER, dispatch, [To, Delivery]) of + {badrpc, Reason} -> + emqx_logger:error("[Broker] Failed to forward msg to ~s: ~p", [Node, Reason]), + Delivery; + Delivery1 -> Delivery1 + end. + +-spec(dispatch(emqx_topic:topic(), emqx_types:delivery()) -> emqx_types:delivery()). +dispatch(Topic, Delivery = #delivery{message = Msg, results = Results}) -> + case subscribers(Topic) of + [] -> + emqx_hooks:run('message.dropped', [#{node => node()}, Msg]), + inc_dropped_cnt(Topic), + Delivery; + [Sub] -> %% optimize? + Cnt = dispatch(Sub, Topic, Msg), + Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]}; + Subs -> + Cnt = lists:foldl( + fun(Sub, Acc) -> + dispatch(Sub, Topic, Msg) + Acc + end, 0, Subs), + Delivery#delivery{results = [{dispatch, Topic, Cnt}|Results]} + end. + +dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> + case erlang:is_process_alive(SubPid) of + true -> + SubPid ! {dispatch, Topic, Msg}, + 1; + false -> 0 + end; +dispatch({shard, I}, Topic, Msg) -> + lists:foldl( + fun(SubPid, Cnt) -> + dispatch(SubPid, Topic, Msg) + Cnt + end, 0, subscribers({shard, Topic, I})). + +inc_dropped_cnt(<<"$SYS/", _/binary>>) -> + ok; +inc_dropped_cnt(_Topic) -> + emqx_metrics:inc('messages/dropped'). + +-spec(subscribers(emqx_topic:topic()) -> [pid()]). +subscribers(Topic) when is_binary(Topic) -> + lookup_value(?SUBSCRIBER, Topic, []); +subscribers(Shard = {shard, _Topic, _I}) -> + lookup_value(?SUBSCRIBER, Shard, []). + +%%------------------------------------------------------------------------------ +%% Subscriber is down +%%------------------------------------------------------------------------------ + +-spec(subscriber_down(pid()) -> true). +subscriber_down(SubPid) -> + lists:foreach( + fun(Topic) -> + case lookup_value(?SUBOPTION, {SubPid, Topic}) of + SubOpts when is_map(SubOpts) -> + _ = emqx_broker_helper:reclaim_seq(Topic), + true = ets:delete(?SUBOPTION, {SubPid, Topic}), + case maps:get(shard, SubOpts, 0) of + 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), + ok = cast(pick(Topic), {unsubscribed, Topic}); + I -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), + ok = cast(pick({Topic, I}), {unsubscribed, Topic, I}) + end; + undefined -> ok + end + end, lookup_value(?SUBSCRIPTION, SubPid, [])), + ets:delete(?SUBSCRIPTION, SubPid). + +%%------------------------------------------------------------------------------ +%% Management APIs +%%------------------------------------------------------------------------------ + +-spec(subscriptions(pid() | emqx_types:subid()) + -> [{emqx_topic:topic(), emqx_types:subopts()}]). +subscriptions(SubPid) when is_pid(SubPid) -> + [{Topic, lookup_value(?SUBOPTION, {SubPid, Topic}, #{})} + || Topic <- lookup_value(?SUBSCRIPTION, SubPid, [])]; +subscriptions(SubId) -> + case emqx_broker_helper:lookup_subpid(SubId) of + SubPid when is_pid(SubPid) -> + subscriptions(SubPid); + undefined -> [] + end. + +-spec(subscribed(pid(), emqx_topic:topic()) -> boolean()). +subscribed(SubPid, Topic) when is_pid(SubPid) -> + ets:member(?SUBOPTION, {SubPid, Topic}); +subscribed(SubId, Topic) when ?is_subid(SubId) -> + SubPid = emqx_broker_helper:lookup_subpid(SubId), + ets:member(?SUBOPTION, {SubPid, Topic}). + +-spec(get_subopts(pid(), emqx_topic:topic()) -> emqx_types:subopts() | undefined). +get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> + lookup_value(?SUBOPTION, {SubPid, Topic}); +get_subopts(SubId, Topic) when ?is_subid(SubId) -> + case emqx_broker_helper:lookup_subpid(SubId) of + SubPid when is_pid(SubPid) -> + get_subopts(SubPid, Topic); + undefined -> undefined + end. + +-spec(set_subopts(emqx_topic:topic(), emqx_types:subopts()) -> boolean()). +set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> + Sub = {self(), Topic}, + case ets:lookup(?SUBOPTION, Sub) of + [{_, OldOpts}] -> + ets:insert(?SUBOPTION, {Sub, maps:merge(OldOpts, NewOpts)}); + [] -> false + end. + +-spec(topics() -> [emqx_topic:topic()]). +topics() -> + emqx_router:topics(). + +%%------------------------------------------------------------------------------ +%% Stats fun +%%------------------------------------------------------------------------------ + +stats_fun() -> + safe_update_stats(?SUBSCRIBER, 'subscribers/count', 'subscribers/max'), + safe_update_stats(?SUBSCRIPTION, 'subscriptions/count', 'subscriptions/max'), + safe_update_stats(?SUBOPTION, 'suboptions/count', 'suboptions/max'). + +safe_update_stats(Tab, Stat, MaxStat) -> + case ets:info(Tab, size) of + undefined -> ok; + Size -> emqx_stats:setstat(Stat, MaxStat, Size) + end. + +%%------------------------------------------------------------------------------ +%% call, cast, pick +%%------------------------------------------------------------------------------ + +call(Broker, Req) -> + gen_server:call(Broker, Req). + +cast(Broker, Msg) -> + gen_server:cast(Broker, Msg). + +%% Pick a broker +pick(Topic) -> + gproc_pool:pick_worker(broker_pool, Topic). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Pool, Id]) -> + true = gproc_pool:connect_worker(Pool, {Pool, Id}), + {ok, #{pool => Pool, id => Id}}. + +handle_call({subscribe, Topic}, _From, State) -> + Ok = emqx_router:do_add_route(Topic), + {reply, Ok, State}; + +handle_call({subscribe, Topic, I}, _From, State) -> + Ok = case get(Shard = {Topic, I}) of + undefined -> + _ = put(Shard, true), + true = ets:insert(?SUBSCRIBER, {Topic, {shard, I}}), + cast(pick(Topic), {subscribe, Topic}); + true -> ok + end, + {reply, Ok, State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[Broker] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({subscribe, Topic}, State) -> + case emqx_router:do_add_route(Topic) of + ok -> ok; + {error, Reason} -> + emqx_logger:error("[Broker] Failed to add route: ~p", [Reason]) + end, + {noreply, State}; + +handle_cast({unsubscribed, Topic}, State) -> + case ets:member(?SUBSCRIBER, Topic) of + false -> + _ = emqx_router:do_delete_route(Topic); + true -> ok + end, + {noreply, State}; + +handle_cast({unsubscribed, Topic, I}, State) -> + case ets:member(?SUBSCRIBER, {shard, Topic, I}) of + false -> + _ = erase({Topic, I}), + true = ets:delete_object(?SUBSCRIBER, {Topic, {shard, I}}), + cast(pick(Topic), {unsubscribed, Topic}); + true -> ok + end, + {noreply, State}; + +handle_cast(Msg, State) -> + emqx_logger:error("[Broker] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + emqx_logger:error("[Broker] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{pool := Pool, id := Id}) -> + gproc_pool:disconnect_worker(Pool, {Pool, Id}). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + diff --git a/src/emqx_broker_helper.erl b/src/emqx_broker_helper.erl new file mode 100644 index 000000000..1ea3f3668 --- /dev/null +++ b/src/emqx_broker_helper.erl @@ -0,0 +1,142 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_broker_helper). + +-behaviour(gen_server). + +-export([start_link/0]). +-export([register_sub/2]). +-export([lookup_subid/1, lookup_subpid/1]). +-export([get_sub_shard/2]). +-export([create_seq/1, reclaim_seq/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(HELPER, ?MODULE). +-define(SUBID, emqx_subid). +-define(SUBMON, emqx_submon). +-define(SUBSEQ, emqx_subseq). +-define(SHARD, 1024). + +-define(BATCH_SIZE, 100000). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?HELPER}, ?MODULE, [], []). + +-spec(register_sub(pid(), emqx_types:subid()) -> ok). +register_sub(SubPid, SubId) when is_pid(SubPid) -> + case ets:lookup(?SUBMON, SubPid) of + [] -> + gen_server:cast(?HELPER, {register_sub, SubPid, SubId}); + [{_, SubId}] -> + ok; + _Other -> + error(subid_conflict) + end. + +-spec(lookup_subid(pid()) -> emqx_types:subid() | undefined). +lookup_subid(SubPid) when is_pid(SubPid) -> + emqx_tables:lookup_value(?SUBMON, SubPid). + +-spec(lookup_subpid(emqx_types:subid()) -> pid()). +lookup_subpid(SubId) -> + emqx_tables:lookup_value(?SUBID, SubId). + +-spec(get_sub_shard(pid(), emqx_topic:topic()) -> non_neg_integer()). +get_sub_shard(SubPid, Topic) -> + case create_seq(Topic) of + Seq when Seq =< ?SHARD -> 0; + _ -> erlang:phash2(SubPid, shards_num()) + 1 + end. + +-spec(shards_num() -> pos_integer()). +shards_num() -> + %% Dynamic sharding later... + ets:lookup_element(?HELPER, shards, 2). + +-spec(create_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). +create_seq(Topic) -> + emqx_sequence:nextval(?SUBSEQ, Topic). + +-spec(reclaim_seq(emqx_topic:topic()) -> emqx_sequence:seqid()). +reclaim_seq(Topic) -> + emqx_sequence:reclaim(?SUBSEQ, Topic). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + %% Helper table + ok = emqx_tables:new(?HELPER, [{read_concurrency, true}]), + %% Shards: CPU * 32 + true = ets:insert(?HELPER, {shards, emqx_vm:schedulers() * 32}), + %% SubSeq: Topic -> SeqId + ok = emqx_sequence:create(?SUBSEQ), + %% SubId: SubId -> SubPid + ok = emqx_tables:new(?SUBID, [public, {read_concurrency, true}, {write_concurrency, true}]), + %% SubMon: SubPid -> SubId + ok = emqx_tables:new(?SUBMON, [public, {read_concurrency, true}, {write_concurrency, true}]), + %% Stats timer + ok = emqx_stats:update_interval(broker_stats, fun emqx_broker:stats_fun/0), + {ok, #{pmon => emqx_pmon:new()}}. + +handle_call(Req, _From, State) -> + emqx_logger:error("[BrokerHelper] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({register_sub, SubPid, SubId}, State = #{pmon := PMon}) -> + true = (SubId =:= undefined) orelse ets:insert(?SUBID, {SubId, SubPid}), + true = ets:insert(?SUBMON, {SubPid, SubId}), + {noreply, State#{pmon := emqx_pmon:monitor(SubPid, PMon)}}; + +handle_cast(Msg, State) -> + emqx_logger:error("[BrokerHelper] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #{pmon := PMon}) -> + SubPids = [SubPid | emqx_misc:drain_down(?BATCH_SIZE)], + ok = emqx_pool:async_submit( + fun lists:foreach/2, [fun clean_down/1, SubPids]), + {_, PMon1} = emqx_pmon:erase_all(SubPids, PMon), + {noreply, State#{pmon := PMon1}}; + +handle_info(Info, State) -> + emqx_logger:error("[BrokerHelper] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + true = emqx_sequence:delete(?SUBSEQ), + emqx_stats:cancel_update(broker_stats). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +clean_down(SubPid) -> + case ets:lookup(?SUBMON, SubPid) of + [{_, SubId}] -> + true = ets:delete(?SUBMON, SubPid), + true = (SubId =:= undefined) + orelse ets:delete_object(?SUBID, {SubId, SubPid}), + emqx_broker:subscriber_down(SubPid); + [] -> ok + end. + diff --git a/src/emqx_broker_sup.erl b/src/emqx_broker_sup.erl new file mode 100644 index 000000000..5b1c0a0e7 --- /dev/null +++ b/src/emqx_broker_sup.erl @@ -0,0 +1,53 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_broker_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%------------------------------------------------------------------------------ +%% Supervisor callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + %% Broker pool + PoolSize = emqx_vm:schedulers() * 2, + BrokerPool = emqx_pool_sup:spec([broker_pool, hash, PoolSize, + {emqx_broker, start_link, []}]), + + %% Shared subscription + SharedSub = #{id => shared_sub, + start => {emqx_shared_sub, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_shared_sub]}, + + %% Broker helper + Helper = #{id => helper, + start => {emqx_broker_helper, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_broker_helper]}, + + {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, Helper]}}. + diff --git a/src/emqx_cli.erl b/src/emqx_cli.erl new file mode 100644 index 000000000..f7d513e9d --- /dev/null +++ b/src/emqx_cli.erl @@ -0,0 +1,33 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_cli). + +-export([print/1, print/2, usage/1, usage/2]). + +print(Msg) -> + io:format(Msg), lists:flatten(io_lib:format("~p", [Msg])). + +print(Format, Args) -> + io:format(Format, Args), lists:flatten(io_lib:format(Format, Args)). + +usage(CmdList) -> + lists:map( + fun({Cmd, Descr}) -> + io:format("~-48s# ~s~n", [Cmd, Descr]), + lists:flatten(io_lib:format("~-48s# ~s~n", [Cmd, Descr])) + end, CmdList). + +usage(Format, Args) -> + usage([{Format, Args}]). diff --git a/src/emqx_client.erl b/src/emqx_client.erl new file mode 100644 index 000000000..2d9cb0a80 --- /dev/null +++ b/src/emqx_client.erl @@ -0,0 +1,1341 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_client). + +-behaviour(gen_statem). + +-include("emqx_mqtt.hrl"). + +-export([start_link/0, start_link/1]). +-export([request/5, request/6, request_async/7, receive_response/3]). +-export([set_request_handler/2, sub_request_topic/3, sub_request_topic/4]). +-export([connect/1]). +-export([subscribe/2, subscribe/3, subscribe/4]). +-export([publish/2, publish/3, publish/4, publish/5]). +-export([unsubscribe/2, unsubscribe/3]). +-export([ping/1]). +-export([disconnect/1, disconnect/2, disconnect/3]). +-export([puback/2, puback/3, puback/4]). +-export([pubrec/2, pubrec/3, pubrec/4]). +-export([pubrel/2, pubrel/3, pubrel/4]). +-export([pubcomp/2, pubcomp/3, pubcomp/4]). +-export([subscriptions/1]). +-export([info/1, stop/1]). +%% For test cases +-export([pause/1, resume/1]). + +-export([initialized/3, waiting_for_connack/3, connected/3]). +-export([init/1, callback_mode/0, handle_event/4, terminate/3, code_change/4]). + +-export_type([client/0, properties/0, payload/0, + pubopt/0, subopt/0, request_input/0, + response_payload/0, request_handler/0, + corr_data/0]). +-export_type([host/0, option/0]). + +%% Default timeout +-define(DEFAULT_KEEPALIVE, 60000). +-define(DEFAULT_ACK_TIMEOUT, 30000). +-define(DEFAULT_CONNECT_TIMEOUT, 60000). + +-define(PROPERTY(Name, Val), #state{properties = #{Name := Val}}). + +-define(WILL_MSG(QoS, Retain, Topic, Props, Payload), + #mqtt_msg{qos = QoS, retain = Retain, topic = Topic, props = Props, payload = Payload}). + +-define(RESPONSE_TIMEOUT_SECONDS, timer:seconds(5)). + +-define(NO_HANDLER, undefined). + +-define(NO_GROUP, <<>>). + +-define(NO_CLIENT_ID, <<>>). + +-type(host() :: inet:ip_address() | inet:hostname()). + +-type corr_data() :: binary(). + +-type(option() :: {name, atom()} + | {owner, pid()} + | {host, host()} + | {hosts, [{host(), inet:port_number()}]} + | {port, inet:port_number()} + | {tcp_opts, [gen_tcp:option()]} + | {ssl, boolean()} + | {ssl_opts, [ssl:ssl_option()]} + | {connect_timeout, pos_integer()} + | {bridge_mode, boolean()} + | {client_id, iodata()} + | {clean_start, boolean()} + | {username, iodata()} + | {password, iodata()} + | {proto_ver, v3 | v4 | v5} + | {keepalive, non_neg_integer()} + | {max_inflight, pos_integer()} + | {retry_interval, timeout()} + | {request_handler, request_handler()} + | {will_topic, iodata()} + | {will_payload, iodata()} + | {will_retain, boolean()} + | {will_qos, qos()} + | {will_props, properties()} + | {auto_ack, boolean()} + | {ack_timeout, pos_integer()} + | {force_ping, boolean()} + | {properties, properties()}). + +-record(mqtt_msg, {qos = ?QOS_0, retain = false, dup = false, + packet_id, topic, props, payload}). + +-type(mqtt_msg() :: #mqtt_msg{}). + +-record(state, {name :: atom(), + owner :: pid(), + host :: host(), + port :: inet:port_number(), + hosts :: [{host(), inet:port_number()}], + socket :: inet:socket(), + sock_opts :: [emqx_client_sock:option()], + connect_timeout :: pos_integer(), + bridge_mode :: boolean(), + client_id :: binary(), + clean_start :: boolean(), + username :: binary() | undefined, + password :: binary() | undefined, + proto_ver :: emqx_mqtt_types:version(), + proto_name :: iodata(), + keepalive :: non_neg_integer(), + keepalive_timer :: reference() | undefined, + force_ping :: boolean(), + paused :: boolean(), + will_flag :: boolean(), + will_msg :: mqtt_msg(), + properties :: properties(), + pending_calls :: list(), + subscriptions :: map(), + max_inflight :: infinity | pos_integer(), + inflight :: emqx_inflight:inflight(), + awaiting_rel :: map(), + auto_ack :: boolean(), + ack_timeout :: pos_integer(), + ack_timer :: reference(), + retry_interval :: pos_integer(), + retry_timer :: reference(), + request_handler :: request_handler(), + session_present :: boolean(), + last_packet_id :: packet_id(), + parse_state :: emqx_frame:state()}). + +-record(call, {id, from, req, ts}). + +-type(client() :: pid() | atom()). + +-type(topic() :: emqx_topic:topic()). + +-type(payload() :: iodata()). + +-type(packet_id() :: emqx_mqtt_types:packet_id()). + +-type(properties() :: emqx_mqtt_types:properties()). + +-type(qos() :: emqx_mqtt_types:qos_name() | emqx_mqtt_types:qos()). + +-type(pubopt() :: {retain, boolean()} | {qos, qos()} | {timeout, timeout()}). + +-type(subopt() :: {rh, 0 | 1 | 2} + | {rap, boolean()} + | {nl, boolean()} + | {qos, qos()}). + +-type(reason_code() :: emqx_mqtt_types:reason_code()). + +-type(subscribe_ret() :: {ok, properties(), [reason_code()]} | {error, term()}). + +-type(request_input() :: binary()). + +-type(response_payload() :: binary()). + +-type(request_handler() :: fun((request_input()) -> response_payload())). + +-type(group() :: binary()). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% @doc Swap in a new request handler on the fly. +-spec(set_request_handler(client(), request_handler()) -> ok). +set_request_handler(Responser, RequestHandler) -> + gen_statem:call(Responser, {set_request_handler, RequestHandler}). + +%% @doc Subscribe to request topic. +-spec(sub_request_topic(client(), qos(), topic()) -> ok). +sub_request_topic(Client, QoS, Topic) -> + sub_request_topic(Client, QoS, Topic, ?NO_GROUP). + +%% @doc Share-subscribe to request topic. +-spec(sub_request_topic(client(), qos(), topic(), group()) -> ok). +sub_request_topic(Client, QoS, Topic, Group) -> + Properties = get_properties(Client), + NewTopic = make_req_rsp_topic(Properties, Topic, Group), + subscribe_req_rsp_topic(Client, QoS, NewTopic). + +-spec(start_link() -> gen_statem:start_ret()). +start_link() -> start_link([]). + +-spec(start_link(map() | [option()]) -> gen_statem:start_ret()). +start_link(Options) when is_map(Options) -> + start_link(maps:to_list(Options)); +start_link(Options) when is_list(Options) -> + ok = emqx_mqtt_props:validate( + proplists:get_value(properties, Options, #{})), + case proplists:get_value(name, Options) of + undefined -> + gen_statem:start_link(?MODULE, [with_owner(Options)], []); + Name when is_atom(Name) -> + gen_statem:start_link({local, Name}, ?MODULE, [with_owner(Options)], []) + end. + +with_owner(Options) -> + case proplists:get_value(owner, Options) of + Owner when is_pid(Owner) -> Options; + undefined -> [{owner, self()} | Options] + end. + +-spec(connect(client()) -> {ok, properties()} | {error, term()}). +connect(Client) -> + gen_statem:call(Client, connect, infinity). + +-spec(subscribe(client(), topic() | {topic(), qos() | [subopt()]} | [{topic(), qos()}]) + -> subscribe_ret()). +subscribe(Client, Topic) when is_binary(Topic) -> + subscribe(Client, {Topic, ?QOS_0}); +subscribe(Client, {Topic, QoS}) when is_binary(Topic), is_atom(QoS) -> + subscribe(Client, {Topic, ?QOS_I(QoS)}); +subscribe(Client, {Topic, QoS}) when is_binary(Topic), ?IS_QOS(QoS) -> + subscribe(Client, [{Topic, ?QOS_I(QoS)}]); +subscribe(Client, Topics) when is_list(Topics) -> + subscribe(Client, #{}, lists:map( + fun({Topic, QoS}) when is_binary(Topic), is_atom(QoS) -> + {Topic, [{qos, ?QOS_I(QoS)}]}; + ({Topic, QoS}) when is_binary(Topic), ?IS_QOS(QoS) -> + {Topic, [{qos, ?QOS_I(QoS)}]}; + ({Topic, Opts}) when is_binary(Topic), is_list(Opts) -> + {Topic, Opts} + end, Topics)). + +-spec(subscribe(client(), topic(), qos() | [subopt()]) -> + subscribe_ret(); + (client(), properties(), [{topic(), qos() | [subopt()]}]) -> + subscribe_ret()). +subscribe(Client, Topic, QoS) when is_binary(Topic), is_atom(QoS) -> + subscribe(Client, Topic, ?QOS_I(QoS)); +subscribe(Client, Topic, QoS) when is_binary(Topic), ?IS_QOS(QoS) -> + subscribe(Client, Topic, [{qos, QoS}]); +subscribe(Client, Topic, Opts) when is_binary(Topic), is_list(Opts) -> + subscribe(Client, #{}, [{Topic, Opts}]); +subscribe(Client, Properties, Topics) when is_map(Properties), is_list(Topics) -> + Topics1 = [{Topic, parse_subopt(Opts)} || {Topic, Opts} <- Topics], + gen_statem:call(Client, {subscribe, Properties, Topics1}). + +-spec(subscribe(client(), properties(), topic(), qos() | [subopt()]) + -> subscribe_ret()). +subscribe(Client, Properties, Topic, QoS) + when is_map(Properties), is_binary(Topic), is_atom(QoS) -> + subscribe(Client, Properties, Topic, ?QOS_I(QoS)); +subscribe(Client, Properties, Topic, QoS) + when is_map(Properties), is_binary(Topic), ?IS_QOS(QoS) -> + subscribe(Client, Properties, Topic, [{qos, QoS}]); +subscribe(Client, Properties, Topic, Opts) + when is_map(Properties), is_binary(Topic), is_list(Opts) -> + subscribe(Client, Properties, [{Topic, Opts}]). + +parse_subopt(Opts) -> + parse_subopt(Opts, #{rh => 0, rap => 0, nl => 0, qos => ?QOS_0}). + +parse_subopt([], Result) -> + Result; +parse_subopt([{rh, I} | Opts], Result) when I >= 0, I =< 2 -> + parse_subopt(Opts, Result#{rh := I}); +parse_subopt([{rap, true} | Opts], Result) -> + parse_subopt(Opts, Result#{rap := 1}); +parse_subopt([{rap, false} | Opts], Result) -> + parse_subopt(Opts, Result#{rap := 0}); +parse_subopt([{nl, true} | Opts], Result) -> + parse_subopt(Opts, Result#{nl := 1}); +parse_subopt([{nl, false} | Opts], Result) -> + parse_subopt(Opts, Result#{nl := 0}); +parse_subopt([{qos, QoS} | Opts], Result) -> + parse_subopt(Opts, Result#{qos := ?QOS_I(QoS)}). + +-spec(request(client(), topic(), topic(), payload(), qos() | [pubopt()]) + -> ok | {ok, packet_id()} | {error, term()}). +request(Client, ResponseTopic, RequestTopic, Payload, QoS) when is_binary(ResponseTopic), is_atom(QoS) -> + request(Client, ResponseTopic, RequestTopic, Payload, [{qos, ?QOS_I(QoS)}]); +request(Client, ResponseTopic, RequestTopic, Payload, QoS) when is_binary(ResponseTopic), ?IS_QOS(QoS) -> + request(Client, ResponseTopic, RequestTopic, Payload, [{qos, QoS}]); +request(Client, ResponseTopic, RequestTopic, Payload, Opts) when is_binary(ResponseTopic), is_list(Opts) -> + request(Client, ResponseTopic, RequestTopic, Payload, Opts, _Properties = #{}). + +%% @doc Send a request to request topic and wait for response. +-spec(request(client(), topic(), topic(), payload(), [pubopt()], properties()) + -> {ok, response_payload()} | {error, term()}). +request(Client, ResponseTopic, RequestTopic, Payload, Opts, Properties) -> + CorrData = make_corr_data(), + case request_async(Client, ResponseTopic, RequestTopic, + Payload, Opts, Properties, CorrData) of + ok -> receive_response(Client, CorrData, Opts); + {error, Reason} -> {error, Reason} + end. + +%% @doc Get client properties. +-spec(get_properties(client()) -> properties()). +get_properties(Client) -> gen_statem:call(Client, get_properties, infinity). + +%% @doc Send a request, but do not wait for response. +%% The caller should expect a `{publish, Response}' message, +%% or call `receive_response/3' to receive the message. +-spec(request_async(client(), topic(), topic(), payload(), + [pubopt()], properties(), corr_data()) -> ok | {error, any()}). +request_async(Client, ResponseTopic, RequestTopic, Payload, Opts, Properties, CorrData) + when is_binary(ResponseTopic), + is_binary(RequestTopic), + is_map(Properties), + is_list(Opts) -> + ok = emqx_mqtt_props:validate(Properties), + Retain = proplists:get_bool(retain, Opts), + QoS = ?QOS_I(proplists:get_value(qos, Opts, ?QOS_0)), + ClientProperties = get_properties(Client), + NewResponseTopic = make_req_rsp_topic(ClientProperties, ResponseTopic), + NewRequestTopic = make_req_rsp_topic(ClientProperties, RequestTopic), + %% This is perhaps not optimal to subscribe the response topic for + %% each and every request even though the response topic is always the same + ok = sub_response_topic(Client, QoS, NewResponseTopic), + NewProperties = maps:merge(Properties, #{'Response-Topic' => NewResponseTopic, + 'Correlation-Data' => CorrData}), + case publish(Client, #mqtt_msg{qos = QoS, + retain = Retain, + topic = NewRequestTopic, + props = NewProperties, + payload = iolist_to_binary(Payload)}) of + ok -> ok; + {ok, _PacketId} -> ok; %% assume auto_ack + {error, Reason} -> {error, Reason} + end. + +%% @doc Block wait the response for a request sent earlier. +-spec(receive_response(client(), corr_data(), [pubopt()]) + -> {ok, response_payload()} | {error, any()}). +receive_response(Client, CorrData, Opts) -> + TimeOut = proplists:get_value(timeout, Opts, ?RESPONSE_TIMEOUT_SECONDS), + MRef = erlang:monitor(process, Client), + TRef = erlang:start_timer(TimeOut, self(), response), + try + receive_response(Client, CorrData, TRef, MRef) + after + erlang:cancel_timer(TRef), + receive {timeout, TRef, _} -> ok after 0 -> ok end, + erlang:demonitor(MRef, [flush]) + end. + +-spec(publish(client(), topic(), payload()) -> ok | {error, term()}). +publish(Client, Topic, Payload) when is_binary(Topic) -> + publish(Client, #mqtt_msg{topic = Topic, qos = ?QOS_0, payload = iolist_to_binary(Payload)}). + +-spec(publish(client(), topic(), payload(), qos() | [pubopt()]) + -> ok | {ok, packet_id()} | {error, term()}). +publish(Client, Topic, Payload, QoS) when is_binary(Topic), is_atom(QoS) -> + publish(Client, Topic, Payload, [{qos, ?QOS_I(QoS)}]); +publish(Client, Topic, Payload, QoS) when is_binary(Topic), ?IS_QOS(QoS) -> + publish(Client, Topic, Payload, [{qos, QoS}]); +publish(Client, Topic, Payload, Opts) when is_binary(Topic), is_list(Opts) -> + publish(Client, Topic, #{}, Payload, Opts). + +-spec(publish(client(), topic(), properties(), payload(), [pubopt()]) + -> ok | {ok, packet_id()} | {error, term()}). +publish(Client, Topic, Properties, Payload, Opts) + when is_binary(Topic), is_map(Properties), is_list(Opts) -> + ok = emqx_mqtt_props:validate(Properties), + Retain = proplists:get_bool(retain, Opts), + QoS = ?QOS_I(proplists:get_value(qos, Opts, ?QOS_0)), + publish(Client, #mqtt_msg{qos = QoS, + retain = Retain, + topic = Topic, + props = Properties, + payload = iolist_to_binary(Payload)}). + +-spec(publish(client(), #mqtt_msg{}) -> ok | {ok, packet_id()} | {error, term()}). +publish(Client, Msg) when is_record(Msg, mqtt_msg) -> + gen_statem:call(Client, {publish, Msg}). + +-spec(unsubscribe(client(), topic() | [topic()]) -> subscribe_ret()). +unsubscribe(Client, Topic) when is_binary(Topic) -> + unsubscribe(Client, [Topic]); +unsubscribe(Client, Topics) when is_list(Topics) -> + unsubscribe(Client, #{}, Topics). + +-spec(unsubscribe(client(), properties(), topic() | [topic()]) -> subscribe_ret()). +unsubscribe(Client, Properties, Topic) when is_map(Properties), is_binary(Topic) -> + unsubscribe(Client, Properties, [Topic]); +unsubscribe(Client, Properties, Topics) when is_map(Properties), is_list(Topics) -> + gen_statem:call(Client, {unsubscribe, Properties, Topics}). + +-spec(ping(client()) -> pong). +ping(Client) -> + gen_statem:call(Client, ping). + +-spec(disconnect(client()) -> ok). +disconnect(Client) -> + disconnect(Client, ?RC_SUCCESS). + +-spec(disconnect(client(), reason_code()) -> ok). +disconnect(Client, ReasonCode) -> + disconnect(Client, ReasonCode, #{}). + +-spec(disconnect(client(), reason_code(), properties()) -> ok). +disconnect(Client, ReasonCode, Properties) -> + gen_statem:call(Client, {disconnect, ReasonCode, Properties}). + +%%------------------------------------------------------------------------------ +%% For test cases +%%------------------------------------------------------------------------------ + +puback(Client, PacketId) when is_integer(PacketId) -> + puback(Client, PacketId, ?RC_SUCCESS). +puback(Client, PacketId, ReasonCode) + when is_integer(PacketId), is_integer(ReasonCode) -> + puback(Client, PacketId, ReasonCode, #{}). +puback(Client, PacketId, ReasonCode, Properties) + when is_integer(PacketId), is_integer(ReasonCode), is_map(Properties) -> + gen_statem:cast(Client, {puback, PacketId, ReasonCode, Properties}). + +pubrec(Client, PacketId) when is_integer(PacketId) -> + pubrec(Client, PacketId, ?RC_SUCCESS). +pubrec(Client, PacketId, ReasonCode) + when is_integer(PacketId), is_integer(ReasonCode) -> + pubrec(Client, PacketId, ReasonCode, #{}). +pubrec(Client, PacketId, ReasonCode, Properties) + when is_integer(PacketId), is_integer(ReasonCode), is_map(Properties) -> + gen_statem:cast(Client, {pubrec, PacketId, ReasonCode, Properties}). + +pubrel(Client, PacketId) when is_integer(PacketId) -> + pubrel(Client, PacketId, ?RC_SUCCESS). +pubrel(Client, PacketId, ReasonCode) + when is_integer(PacketId), is_integer(ReasonCode) -> + pubrel(Client, PacketId, ReasonCode, #{}). +pubrel(Client, PacketId, ReasonCode, Properties) + when is_integer(PacketId), is_integer(ReasonCode), is_map(Properties) -> + gen_statem:cast(Client, {pubrel, PacketId, ReasonCode, Properties}). + +pubcomp(Client, PacketId) when is_integer(PacketId) -> + pubcomp(Client, PacketId, ?RC_SUCCESS). +pubcomp(Client, PacketId, ReasonCode) + when is_integer(PacketId), is_integer(ReasonCode) -> + pubcomp(Client, PacketId, ReasonCode, #{}). +pubcomp(Client, PacketId, ReasonCode, Properties) + when is_integer(PacketId), is_integer(ReasonCode), is_map(Properties) -> + gen_statem:cast(Client, {pubcomp, PacketId, ReasonCode, Properties}). + +subscriptions(Client) -> + gen_statem:call(Client, subscriptions). + +info(Client) -> + gen_statem:call(Client, info). + +stop(Client) -> + gen_statem:call(Client, stop). + +pause(Client) -> + gen_statem:call(Client, pause). + +resume(Client) -> + gen_statem:call(Client, resume). + +%%------------------------------------------------------------------------------ +%% gen_statem callbacks +%%------------------------------------------------------------------------------ + +init([Options]) -> + process_flag(trap_exit, true), + ClientId = case {proplists:get_value(proto_ver, Options, v4), + proplists:get_value(client_id, Options)} of + {v5, undefined} -> ?NO_CLIENT_ID; + {_ver, undefined} -> random_client_id(); + {_ver, Id} -> iolist_to_binary(Id) + end, + State = init(Options, #state{host = {127,0,0,1}, + port = 1883, + hosts = [], + sock_opts = [], + bridge_mode = false, + client_id = ClientId, + clean_start = true, + proto_ver = ?MQTT_PROTO_V4, + proto_name = <<"MQTT">>, + keepalive = ?DEFAULT_KEEPALIVE, + force_ping = false, + paused = false, + will_flag = false, + will_msg = #mqtt_msg{}, + pending_calls = [], + subscriptions = #{}, + max_inflight = infinity, + inflight = emqx_inflight:new(0), + awaiting_rel = #{}, + properties = #{}, + auto_ack = true, + ack_timeout = ?DEFAULT_ACK_TIMEOUT, + retry_interval = 0, + request_handler = ?NO_HANDLER, + connect_timeout = ?DEFAULT_CONNECT_TIMEOUT, + last_packet_id = 1}), + {ok, initialized, init_parse_state(State)}. + +random_client_id() -> + rand:seed(exsplus, erlang:timestamp()), + I1 = rand:uniform(round(math:pow(2, 48))) - 1, + I2 = rand:uniform(round(math:pow(2, 32))) - 1, + {ok, Host} = inet:gethostname(), + iolist_to_binary(["emqx-client-", Host, "-", io_lib:format("~12.16.0b~8.16.0b", [I1, I2])]). + +init([], State) -> + State; +init([{name, Name} | Opts], State) -> + init(Opts, State#state{name = Name}); +init([{owner, Owner} | Opts], State) when is_pid(Owner) -> + link(Owner), + init(Opts, State#state{owner = Owner}); +init([{host, Host} | Opts], State) -> + init(Opts, State#state{host = Host}); +init([{port, Port} | Opts], State) -> + init(Opts, State#state{port = Port}); +init([{hosts, Hosts} | Opts], State) -> + Hosts1 = + lists:foldl(fun({Host, Port}, Acc) -> + [{Host, Port}|Acc]; + (Host, Acc) -> + [{Host, 1883}|Acc] + end, [], Hosts), + init(Opts, State#state{hosts = Hosts1}); +init([{tcp_opts, TcpOpts} | Opts], State = #state{sock_opts = SockOpts}) -> + init(Opts, State#state{sock_opts = emqx_misc:merge_opts(SockOpts, TcpOpts)}); +init([{ssl, EnableSsl} | Opts], State) -> + case lists:keytake(ssl_opts, 1, Opts) of + {value, SslOpts, WithOutSslOpts} -> + init([SslOpts, {ssl, EnableSsl}| WithOutSslOpts], State); + false -> + init([{ssl_opts, []}, {ssl, EnableSsl}| Opts], State) + end; +init([{ssl_opts, SslOpts} | Opts], State = #state{sock_opts = SockOpts}) -> + case lists:keytake(ssl, 1, Opts) of + {value, {ssl, true}, WithOutEnableSsl} -> + ok = ssl:start(), + SockOpts1 = emqx_misc:merge_opts(SockOpts, [{ssl_opts, SslOpts}]), + init(WithOutEnableSsl, State#state{sock_opts = SockOpts1}); + {value, {ssl, false}, WithOutEnableSsl} -> + init(WithOutEnableSsl, State); + false -> + init(Opts, State) + end; +init([{client_id, ClientId} | Opts], State) -> + init(Opts, State#state{client_id = iolist_to_binary(ClientId)}); +init([{clean_start, CleanStart} | Opts], State) when is_boolean(CleanStart) -> + init(Opts, State#state{clean_start = CleanStart}); +init([{username, Username} | Opts], State) -> + init(Opts, State#state{username = iolist_to_binary(Username)}); +init([{password, Password} | Opts], State) -> + init(Opts, State#state{password = iolist_to_binary(Password)}); +init([{keepalive, Secs} | Opts], State) -> + init(Opts, State#state{keepalive = timer:seconds(Secs)}); +init([{proto_ver, v3} | Opts], State) -> + init(Opts, State#state{proto_ver = ?MQTT_PROTO_V3, + proto_name = <<"MQIsdp">>}); +init([{proto_ver, v4} | Opts], State) -> + init(Opts, State#state{proto_ver = ?MQTT_PROTO_V4, + proto_name = <<"MQTT">>}); +init([{proto_ver, v5} | Opts], State) -> + init(Opts, State#state{proto_ver = ?MQTT_PROTO_V5, + proto_name = <<"MQTT">>}); +init([{will_topic, Topic} | Opts], State = #state{will_msg = WillMsg}) -> + WillMsg1 = init_will_msg({topic, Topic}, WillMsg), + init(Opts, State#state{will_flag = true, will_msg = WillMsg1}); +init([{will_props, Properties} | Opts], State = #state{will_msg = WillMsg}) -> + init(Opts, State#state{will_msg = init_will_msg({props, Properties}, WillMsg)}); +init([{will_payload, Payload} | Opts], State = #state{will_msg = WillMsg}) -> + init(Opts, State#state{will_msg = init_will_msg({payload, Payload}, WillMsg)}); +init([{will_retain, Retain} | Opts], State = #state{will_msg = WillMsg}) -> + init(Opts, State#state{will_msg = init_will_msg({retain, Retain}, WillMsg)}); +init([{will_qos, QoS} | Opts], State = #state{will_msg = WillMsg}) -> + init(Opts, State#state{will_msg = init_will_msg({qos, QoS}, WillMsg)}); +init([{connect_timeout, Timeout}| Opts], State) -> + init(Opts, State#state{connect_timeout = timer:seconds(Timeout)}); +init([{ack_timeout, Timeout}| Opts], State) -> + init(Opts, State#state{ack_timeout = timer:seconds(Timeout)}); +init([force_ping | Opts], State) -> + init(Opts, State#state{force_ping = true}); +init([{force_ping, ForcePing} | Opts], State) when is_boolean(ForcePing) -> + init(Opts, State#state{force_ping = ForcePing}); +init([{properties, Properties} | Opts], State = #state{properties = InitProps}) -> + init(Opts, State#state{properties = maps:merge(InitProps, Properties)}); +init([{max_inflight, infinity} | Opts], State) -> + init(Opts, State#state{max_inflight = infinity, + inflight = emqx_inflight:new(0)}); +init([{max_inflight, I} | Opts], State) when is_integer(I) -> + init(Opts, State#state{max_inflight = I, + inflight = emqx_inflight:new(I)}); +init([auto_ack | Opts], State) -> + init(Opts, State#state{auto_ack = true}); +init([{auto_ack, AutoAck} | Opts], State) when is_boolean(AutoAck) -> + init(Opts, State#state{auto_ack = AutoAck}); +init([{retry_interval, I} | Opts], State) -> + init(Opts, State#state{retry_interval = timer:seconds(I)}); +init([{request_handler, Handler} | Opts], State) -> + init(Opts, State#state{request_handler = Handler}); +init([{bridge_mode, Mode} | Opts], State) when is_boolean(Mode) -> + init(Opts, State#state{bridge_mode = Mode}); +init([_Opt | Opts], State) -> + init(Opts, State). + +init_will_msg({topic, Topic}, WillMsg) -> + WillMsg#mqtt_msg{topic = iolist_to_binary(Topic)}; +init_will_msg({props, Props}, WillMsg) -> + WillMsg#mqtt_msg{props = Props}; +init_will_msg({payload, Payload}, WillMsg) -> + WillMsg#mqtt_msg{payload = iolist_to_binary(Payload)}; +init_will_msg({retain, Retain}, WillMsg) when is_boolean(Retain) -> + WillMsg#mqtt_msg{retain = Retain}; +init_will_msg({qos, QoS}, WillMsg) -> + WillMsg#mqtt_msg{qos = ?QOS_I(QoS)}. + +init_parse_state(State = #state{proto_ver = Ver, properties = Properties}) -> + Size = maps:get('Maximum-Packet-Size', Properties, ?MAX_PACKET_SIZE), + State#state{parse_state = emqx_frame:initial_state( + #{max_packet_size => Size, version => Ver})}. + +callback_mode() -> state_functions. + +initialized({call, From}, connect, State = #state{sock_opts = SockOpts, + connect_timeout = Timeout}) -> + case sock_connect(hosts(State), SockOpts, Timeout) of + {ok, Sock} -> + case mqtt_connect(run_sock(State#state{socket = Sock})) of + {ok, NewState} -> + {next_state, waiting_for_connack, + add_call(new_call(connect, From), NewState), [Timeout]}; + Error = {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, Error}]} + end; + Error = {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, Error}]} + end; + +initialized(EventType, EventContent, State) -> + handle_event(EventType, EventContent, initialized, State). + +mqtt_connect(State = #state{client_id = ClientId, + clean_start = CleanStart, + bridge_mode = IsBridge, + username = Username, + password = Password, + proto_ver = ProtoVer, + proto_name = ProtoName, + keepalive = KeepAlive, + will_flag = WillFlag, + will_msg = WillMsg, + properties = Properties}) -> + ?WILL_MSG(WillQoS, WillRetain, WillTopic, WillProps, WillPayload) = WillMsg, + ConnProps = emqx_mqtt_props:filter(?CONNECT, Properties), + send(?CONNECT_PACKET( + #mqtt_packet_connect{proto_ver = ProtoVer, + proto_name = ProtoName, + is_bridge = IsBridge, + clean_start = CleanStart, + will_flag = WillFlag, + will_qos = WillQoS, + will_retain = WillRetain, + keepalive = KeepAlive, + properties = ConnProps, + client_id = ClientId, + will_props = WillProps, + will_topic = WillTopic, + will_payload = WillPayload, + username = Username, + password = Password}), State). + +waiting_for_connack(cast, ?CONNACK_PACKET(?RC_SUCCESS, + SessPresent, + Properties), + State = #state{properties = AllProps, + client_id = ClientId}) -> + case take_call(connect, State) of + {value, #call{from = From}, State1} -> + AllProps1 = case Properties of + undefined -> AllProps; + _ -> maps:merge(AllProps, Properties) + end, + Reply = {ok, Properties}, + State2 = State1#state{client_id = assign_id(ClientId, AllProps1), + properties = AllProps1, + session_present = SessPresent}, + {next_state, connected, ensure_keepalive_timer(State2), + [{reply, From, Reply}]}; + false -> + {stop, bad_connack} + end; + +waiting_for_connack(cast, ?CONNACK_PACKET(ReasonCode, + _SessPresent, + Properties), State = #state{ proto_ver = ProtoVer}) -> + Reason = emqx_reason_codes:name(ReasonCode, ProtoVer), + case take_call(connect, State) of + {value, #call{from = From}, _State} -> + Reply = {error, {Reason, Properties}}, + {stop_and_reply, Reason, [{reply, From, Reply}]}; + false -> {stop, connack_error} + end; + +waiting_for_connack(timeout, _Timeout, State) -> + case take_call(connect, State) of + {value, #call{from = From}, _State} -> + Reply = {error, connack_timeout}, + {stop_and_reply, connack_timeout, [{reply, From, Reply}]}; + false -> {stop, connack_timeout} + end; + +waiting_for_connack(EventType, EventContent, State) -> + handle_event(EventType, EventContent, waiting_for_connack, State). + +connected({call, From}, subscriptions, State = #state{subscriptions = Subscriptions}) -> + {keep_state, State, [{reply, From, maps:to_list(Subscriptions)}]}; + +connected({call, From}, info, State) -> + Info = lists:zip(record_info(fields, state), tl(tuple_to_list(State))), + {keep_state, State, [{reply, From, Info}]}; + +connected({call, From}, pause, State) -> + {keep_state, State#state{paused = true}, [{reply, From, ok}]}; + +connected({call, From}, resume, State) -> + {keep_state, State#state{paused = false}, [{reply, From, ok}]}; + +connected({call, From}, stop, _State) -> + {stop_and_reply, normal, [{reply, From, ok}]}; + +connected({call, From}, get_properties, State = #state{properties = Properties}) -> + {keep_state, State, [{reply, From, Properties}]}; + +connected({call, From}, client_id, State = #state{client_id = ClientId}) -> + {keep_state, State, [{reply, From, ClientId}]}; + +connected({call, From}, {set_request_handler, RequestHandler}, State) -> + {keep_state, State#state{request_handler = RequestHandler}, [{reply, From, ok}]}; + +connected({call, From}, SubReq = {subscribe, Properties, Topics}, + State = #state{last_packet_id = PacketId, subscriptions = Subscriptions}) -> + case send(?SUBSCRIBE_PACKET(PacketId, Properties, Topics), State) of + {ok, NewState} -> + Call = new_call({subscribe, PacketId}, From, SubReq), + Subscriptions1 = + lists:foldl(fun({Topic, Opts}, Acc) -> + maps:put(Topic, Opts, Acc) + end, Subscriptions, Topics), + {keep_state, ensure_ack_timer(add_call(Call,NewState#state{subscriptions = Subscriptions1}))}; + Error = {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, Error}]} + end; + +connected({call, From}, {publish, Msg = #mqtt_msg{qos = ?QOS_0}}, State) -> + case send(Msg, State) of + {ok, NewState} -> + {keep_state, NewState, [{reply, From, ok}]}; + Error = {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, Error}]} + end; + +connected({call, From}, {publish, Msg = #mqtt_msg{qos = QoS}}, + State = #state{inflight = Inflight, last_packet_id = PacketId}) + when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2) -> + case emqx_inflight:is_full(Inflight) of + true -> + {keep_state, State, [{reply, From, {error, inflight_full}}]}; + false -> + Msg1 = Msg#mqtt_msg{packet_id = PacketId}, + case send(Msg1, State) of + {ok, NewState} -> + Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg1, os:timestamp()}, Inflight), + {keep_state, ensure_retry_timer(NewState#state{inflight = Inflight1}), + [{reply, From, {ok, PacketId}}]}; + Error = {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, Error}]} + end + end; + +connected({call, From}, UnsubReq = {unsubscribe, Properties, Topics}, + State = #state{last_packet_id = PacketId}) -> + case send(?UNSUBSCRIBE_PACKET(PacketId, Properties, Topics), State) of + {ok, NewState} -> + Call = new_call({unsubscribe, PacketId}, From, UnsubReq), + {keep_state, ensure_ack_timer(add_call(Call, NewState))}; + Error = {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, Error}]} + end; + +connected({call, From}, ping, State) -> + case send(?PACKET(?PINGREQ), State) of + {ok, NewState} -> + Call = new_call(ping, From), + {keep_state, ensure_ack_timer(add_call(Call, NewState))}; + Error = {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, Error}]} + end; + +connected({call, From}, {disconnect, ReasonCode, Properties}, State) -> + case send(?DISCONNECT_PACKET(ReasonCode, Properties), State) of + {ok, NewState} -> + {stop_and_reply, normal, [{reply, From, ok}], NewState}; + Error = {error, Reason} -> + {stop_and_reply, Reason, [{reply, From, Error}]} + end; + +connected(cast, {puback, PacketId, ReasonCode, Properties}, State) -> + send_puback(?PUBACK_PACKET(PacketId, ReasonCode, Properties), State); + +connected(cast, {pubrec, PacketId, ReasonCode, Properties}, State) -> + send_puback(?PUBREC_PACKET(PacketId, ReasonCode, Properties), State); + +connected(cast, {pubrel, PacketId, ReasonCode, Properties}, State) -> + send_puback(?PUBREL_PACKET(PacketId, ReasonCode, Properties), State); + +connected(cast, {pubcomp, PacketId, ReasonCode, Properties}, State) -> + send_puback(?PUBCOMP_PACKET(PacketId, ReasonCode, Properties), State); + +connected(cast, ?PUBLISH_PACKET(_QoS, _PacketId), State = #state{paused = true}) -> + {keep_state, State}; + +connected(cast, Packet = ?PUBLISH_PACKET(?QOS_0, _Topic, _PacketId, Properties, Payload), + State) when Properties =/= undefined -> + NewState = response_publish(Properties, State, ?QOS_0, Payload), + {keep_state, deliver(packet_to_msg(Packet), NewState)}; +connected(cast, Packet = ?PUBLISH_PACKET(?QOS_0, _PacketId), State) -> + {keep_state, deliver(packet_to_msg(Packet), State)}; + +connected(cast, Packet = ?PUBLISH_PACKET(?QOS_1, _Topic, _PacketId, Properties, Payload), State) + when Properties =/= undefined -> + NewState = response_publish(Properties, State, ?QOS_1, Payload), + publish_process(?QOS_1, Packet, NewState); + +connected(cast, Packet = ?PUBLISH_PACKET(?QOS_1, _PacketId), State) -> + publish_process(?QOS_1, Packet, State); + +connected(cast, Packet = ?PUBLISH_PACKET(?QOS_2, _Topic, _PacketId, Properties, Payload), State) + when Properties =/= undefined -> + NewState = response_publish(Properties, State, ?QOS_2, Payload), + publish_process(?QOS_2, Packet, NewState); +connected(cast, Packet = ?PUBLISH_PACKET(?QOS_2, _PacketId), State) -> + publish_process(?QOS_2, Packet, State); + +connected(cast, ?PUBACK_PACKET(PacketId, ReasonCode, Properties), + State = #state{owner = Owner, inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, {publish, #mqtt_msg{packet_id = PacketId}, _Ts}} -> + Owner ! {puback, #{packet_id => PacketId, + reason_code => ReasonCode, + properties => Properties}}, + {keep_state, State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}}; + none -> + emqx_logger:warning("Unexpected PUBACK: ~p", [PacketId]), + {keep_state, State} + end; + +connected(cast, ?PUBREC_PACKET(PacketId), State = #state{inflight = Inflight}) -> + send_puback(?PUBREL_PACKET(PacketId), + case emqx_inflight:lookup(PacketId, Inflight) of + {value, {publish, _Msg, _Ts}} -> + Inflight1 = emqx_inflight:update(PacketId, {pubrel, PacketId, os:timestamp()}, Inflight), + State#state{inflight = Inflight1}; + {value, {pubrel, _Ref, _Ts}} -> + emqx_logger:warning("Duplicated PUBREC Packet: ~p", [PacketId]), + State; + none -> + emqx_logger:warning("Unexpected PUBREC Packet: ~p", [PacketId]), + State + end); + +%%TODO::... if auto_ack is false, should we take PacketId from the map? +connected(cast, ?PUBREL_PACKET(PacketId), + State = #state{awaiting_rel = AwaitingRel, auto_ack = AutoAck}) -> + case maps:take(PacketId, AwaitingRel) of + {Packet, AwaitingRel1} -> + NewState = deliver(packet_to_msg(Packet), State#state{awaiting_rel = AwaitingRel1}), + case AutoAck of + true -> send_puback(?PUBCOMP_PACKET(PacketId), NewState); + false -> {keep_state, NewState} + end; + error -> + emqx_logger:warning("Unexpected PUBREL: ~p", [PacketId]), + {keep_state, State} + end; + +connected(cast, ?PUBCOMP_PACKET(PacketId, ReasonCode, Properties), + State = #state{owner = Owner, inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, {pubrel, _PacketId, _Ts}} -> + Owner ! {puback, #{packet_id => PacketId, + reason_code => ReasonCode, + properties => Properties}}, + {keep_state, State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}}; + none -> + emqx_logger:warning("Unexpected PUBCOMP Packet: ~p", [PacketId]), + {keep_state, State} + end; + +connected(cast, ?SUBACK_PACKET(PacketId, Properties, ReasonCodes), + State = #state{subscriptions = _Subscriptions}) -> + case take_call({subscribe, PacketId}, State) of + {value, #call{from = From}, NewState} -> + %%TODO: Merge reason codes to subscriptions? + Reply = {ok, Properties, ReasonCodes}, + {keep_state, NewState, [{reply, From, Reply}]}; + false -> {keep_state, State} + end; + +connected(cast, ?UNSUBACK_PACKET(PacketId, Properties, ReasonCodes), + State = #state{subscriptions = Subscriptions}) -> + case take_call({unsubscribe, PacketId}, State) of + {value, #call{from = From, req = {_, _, Topics}}, NewState} -> + Subscriptions1 = + lists:foldl(fun(Topic, Acc) -> + maps:remove(Topic, Acc) + end, Subscriptions, Topics), + {keep_state, NewState#state{subscriptions = Subscriptions1}, + [{reply, From, {ok, Properties, ReasonCodes}}]}; + false -> {keep_state, State} + end; + +connected(cast, ?PACKET(?PINGRESP), State = #state{pending_calls = []}) -> + {keep_state, State}; +connected(cast, ?PACKET(?PINGRESP), State) -> + case take_call(ping, State) of + {value, #call{from = From}, NewState} -> + {keep_state, NewState, [{reply, From, pong}]}; + false -> {keep_state, State} + end; + +connected(cast, ?DISCONNECT_PACKET(ReasonCode, Properties), + State = #state{owner = Owner}) -> + Owner ! {disconnected, ReasonCode, Properties}, + {stop, disconnected, State}; + +connected(info, {timeout, _TRef, keepalive}, State = #state{force_ping = true}) -> + case send(?PACKET(?PINGREQ), State) of + {ok, NewState} -> + {keep_state, ensure_keepalive_timer(NewState)}; + Error -> {stop, Error} + end; + +connected(info, {timeout, TRef, keepalive}, + State = #state{socket = Sock, paused = Paused, keepalive_timer = TRef}) -> + case (not Paused) andalso should_ping(Sock) of + true -> + case send(?PACKET(?PINGREQ), State) of + {ok, NewState} -> + {keep_state, ensure_keepalive_timer(NewState), [hibernate]}; + Error -> {stop, Error} + end; + false -> + {keep_state, ensure_keepalive_timer(State), [hibernate]}; + {error, Reason} -> + {stop, Reason} + end; + +connected(info, {timeout, TRef, ack}, State = #state{ack_timer = TRef, + ack_timeout = Timeout, + pending_calls = Calls}) -> + NewState = State#state{ack_timer = undefined, + pending_calls = timeout_calls(Timeout, Calls)}, + {keep_state, ensure_ack_timer(NewState)}; + +connected(info, {timeout, TRef, retry}, State = #state{retry_timer = TRef, + inflight = Inflight}) -> + case emqx_inflight:is_empty(Inflight) of + true -> {keep_state, State#state{retry_timer = undefined}}; + false -> retry_send(State) + end; + +connected(EventType, EventContent, Data) -> + handle_event(EventType, EventContent, connected, Data). + +should_ping(Sock) -> + case emqx_client_sock:getstat(Sock, [send_oct]) of + {ok, [{send_oct, Val}]} -> + OldVal = get(send_oct), put(send_oct, Val), + OldVal == undefined orelse OldVal == Val; + Error = {error, _Reason} -> + Error + end. + +handle_event(info, {TcpOrSsL, _Sock, Data}, _StateName, State) + when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl -> + emqx_logger:debug("RECV Data: ~p", [Data]), + receive_loop(Data, run_sock(State)); + +handle_event(info, {Error, _Sock, Reason}, _StateName, State) + when Error =:= tcp_error; Error =:= ssl_error -> + {stop, Reason, State}; + +handle_event(info, {Closed, _Sock}, _StateName, State) + when Closed =:= tcp_closed; Closed =:= ssl_closed -> + {stop, {shutdown, Closed}, State}; + +handle_event(info, {'EXIT', Owner, Reason}, _, #state{owner = Owner}) -> + {stop, Reason}; + +handle_event(info, {inet_reply, _Sock, ok}, _, State) -> + {keep_state, State}; + +handle_event(info, {inet_reply, _Sock, {error, Reason}}, _, State) -> + {stop, Reason, State}; + +handle_event(EventType, EventContent, StateName, StateData) -> + emqx_logger:error("State: ~s, Unexpected Event: (~p, ~p)", + [StateName, EventType, EventContent]), + {keep_state, StateData}. + +%% Mandatory callback functions +terminate(_Reason, _State, #state{socket = undefined}) -> + ok; +terminate(_Reason, _State, #state{socket = Socket}) -> + emqx_client_sock:close(Socket). + +code_change(_Vsn, State, Data, _Extra) -> + {ok, State, Data}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +%% Subscribe to response topic. +-spec(sub_response_topic(client(), qos(), topic()) -> ok). +sub_response_topic(Client, QoS, Topic) when is_binary(Topic) -> + subscribe_req_rsp_topic(Client, QoS, Topic). + +receive_response(Client, CorrData, TRef, MRef) -> + receive + {publish, Response} -> + {ok, Properties} = maps:find(properties, Response), + case maps:find('Correlation-Data', Properties) of + {ok, CorrData} -> + maps:find(payload, Response); + _ -> + emqx_logger:debug("Discarded stale response: ~p", [Response]), + receive_response(Client, CorrData, TRef, MRef) + end; + {timeout, TRef, response} -> + {error, timeout}; + {'DOWN', MRef, process, _, _} -> + {error, client_down} + end. + +%% Make a unique correlation data for each request. +%% It has to be unique because stale responses should be discarded. +make_corr_data() -> term_to_binary(make_ref()). + +%% Shared function for request and response topic subscription. +subscribe_req_rsp_topic(Client, QoS, Topic) -> + %% It is a Protocol Error to set the No Local bit to 1 on a Shared Subscription + {ok, _Props, _QoS} = subscribe(Client, [{Topic, [{rh, 2}, {rap, false}, + {nl, not ?IS_SHARE(Topic)}, + {qos, QoS}]}]), + emqx_logger:debug("Subscribed to topic ~s", [Topic]), + ok. + +%% Make a request or response topic. +make_req_rsp_topic(Properties, Topic) -> + make_req_rsp_topic(Properties, Topic, ?NO_GROUP). + +%% Same as make_req_rsp_topic/2, but allow shared subscription (for request topics) +make_req_rsp_topic(Properties, Topic, Group) -> + case maps:find('Response-Information', Properties) of + {ok, ResponseInformation} when ResponseInformation =/= <<>> -> + emqx_topic:join([req_rsp_topic_prefix(Group, ResponseInformation), Topic]); + _ -> + erlang:error(no_response_information) + end. + +req_rsp_topic_prefix(?NO_GROUP, Prefix) -> Prefix; +req_rsp_topic_prefix(Group, Prefix) -> ?SHARE(Group, Prefix). + +assign_id(?NO_CLIENT_ID, Props) -> + case maps:find('Assigned-Client-Identifier', Props) of + {ok, Value} -> + Value; + _ -> + error(bad_client_id) + end; +assign_id(Id, _Props) -> + Id. + +publish_process(?QOS_1, Packet = ?PUBLISH_PACKET(?QOS_1, PacketId), State = #state{auto_ack = AutoAck}) -> + _ = deliver(packet_to_msg(Packet), State), + case AutoAck of + true -> send_puback(?PUBACK_PACKET(PacketId), State); + false -> {keep_state, State} + end; +publish_process(?QOS_2, Packet = ?PUBLISH_PACKET(?QOS_2, PacketId), + State = #state{awaiting_rel = AwaitingRel}) -> + case send_puback(?PUBREC_PACKET(PacketId), State) of + {keep_state, NewState} -> + AwaitingRel1 = maps:put(PacketId, Packet, AwaitingRel), + {keep_state, NewState#state{awaiting_rel = AwaitingRel1}}; + Stop -> Stop + end. + +response_publish(undefined, State, _QoS, _Payload) -> + State; +response_publish(Properties, State = #state{request_handler = RequestHandler}, QoS, Payload) -> + case maps:find('Response-Topic', Properties) of + {ok, ResponseTopic} -> + case RequestHandler of + ?NO_HANDLER -> State; + _ -> do_publish(ResponseTopic, Properties, State, QoS, Payload) + end; + _ -> + State + end. + +do_publish(ResponseTopic, Properties, State = #state{request_handler = RequestHandler}, ?QOS_0, Payload) -> + Msg = #mqtt_msg{qos = ?QOS_0, + retain = false, + topic = ResponseTopic, + props = Properties, + payload = RequestHandler(Payload) + }, + case send(Msg, State) of + {ok, NewState} -> NewState; + _Error -> State + end; +do_publish(ResponseTopic, Properties, State = #state{request_handler = RequestHandler, + inflight = Inflight, + last_packet_id = PacketId}, + QoS, Payload) + when (QoS =:= ?QOS_1); (QoS =:= ?QOS_2)-> + case emqx_inflight:is_full(Inflight) of + true -> + emqx_logger:error("Inflight is full"), + State; + false -> + Msg = #mqtt_msg{packet_id = PacketId, + qos = QoS, + retain = false, + topic = ResponseTopic, + props = Properties, + payload = RequestHandler(Payload)}, + case send(Msg, State) of + {ok, NewState} -> + Inflight1 = emqx_inflight:insert(PacketId, {publish, Msg, os:timestamp()}, Inflight), + ensure_retry_timer(NewState#state{inflight = Inflight1}); + {error, Reason} -> + emqx_logger:error("Send failed: ~p", [Reason]), + State + end + end. + +ensure_keepalive_timer(State = ?PROPERTY('Server-Keep-Alive', Secs)) -> + ensure_keepalive_timer(timer:seconds(Secs), State); +ensure_keepalive_timer(State = #state{keepalive = 0}) -> + State; +ensure_keepalive_timer(State = #state{keepalive = I}) -> + ensure_keepalive_timer(I, State). +ensure_keepalive_timer(I, State) when is_integer(I) -> + State#state{keepalive_timer = erlang:start_timer(I, self(), keepalive)}. + +new_call(Id, From) -> + new_call(Id, From, undefined). +new_call(Id, From, Req) -> + #call{id = Id, from = From, req = Req, ts = os:timestamp()}. + +add_call(Call, Data = #state{pending_calls = Calls}) -> + Data#state{pending_calls = [Call | Calls]}. + +take_call(Id, Data = #state{pending_calls = Calls}) -> + case lists:keytake(Id, #call.id, Calls) of + {value, Call, Left} -> + {value, Call, Data#state{pending_calls = Left}}; + false -> false + end. + +timeout_calls(Timeout, Calls) -> + timeout_calls(os:timestamp(), Timeout, Calls). +timeout_calls(Now, Timeout, Calls) -> + lists:foldl(fun(C = #call{from = From, ts = Ts}, Acc) -> + case (timer:now_diff(Now, Ts) div 1000) >= Timeout of + true -> gen_statem:reply(From, {error, ack_timeout}), + Acc; + false -> [C | Acc] + end + end, [], Calls). + +ensure_ack_timer(State = #state{ack_timer = undefined, + ack_timeout = Timeout, + pending_calls = Calls}) when length(Calls) > 0 -> + State#state{ack_timer = erlang:start_timer(Timeout, self(), ack)}; +ensure_ack_timer(State) -> State. + +ensure_retry_timer(State = #state{retry_interval = Interval}) -> + ensure_retry_timer(Interval, State). +ensure_retry_timer(Interval, State = #state{retry_timer = undefined}) + when Interval > 0 -> + State#state{retry_timer = erlang:start_timer(Interval, self(), retry)}; +ensure_retry_timer(_Interval, State) -> + State. + +retry_send(State = #state{inflight = Inflight}) -> + SortFun = fun({_, _, Ts1}, {_, _, Ts2}) -> Ts1 < Ts2 end, + Msgs = lists:sort(SortFun, emqx_inflight:values(Inflight)), + retry_send(Msgs, os:timestamp(), State ). + +retry_send([], _Now, State) -> + {keep_state, ensure_retry_timer(State)}; +retry_send([{Type, Msg, Ts} | Msgs], Now, State = #state{retry_interval = Interval}) -> + Diff = timer:now_diff(Now, Ts) div 1000, %% micro -> ms + case (Diff >= Interval) of + true -> case retry_send(Type, Msg, Now, State) of + {ok, NewState} -> retry_send(Msgs, Now, NewState); + {error, Error} -> {stop, Error} + end; + false -> {keep_state, ensure_retry_timer(Interval - Diff, State)} + end. + +retry_send(publish, Msg = #mqtt_msg{qos = QoS, packet_id = PacketId}, + Now, State = #state{inflight = Inflight}) -> + Msg1 = Msg#mqtt_msg{dup = (QoS =:= ?QOS_1)}, + case send(Msg1, State) of + {ok, NewState} -> + Inflight1 = emqx_inflight:update(PacketId, {publish, Msg1, Now}, Inflight), + {ok, NewState#state{inflight = Inflight1}}; + Error = {error, _Reason} -> + Error + end; +retry_send(pubrel, PacketId, Now, State = #state{inflight = Inflight}) -> + case send(?PUBREL_PACKET(PacketId), State) of + {ok, NewState} -> + Inflight1 = emqx_inflight:update(PacketId, {pubrel, PacketId, Now}, Inflight), + {ok, NewState#state{inflight = Inflight1}}; + Error = {error, _Reason} -> + Error + end. + +deliver(#mqtt_msg{qos = QoS, dup = Dup, retain = Retain, packet_id = PacketId, + topic = Topic, props = Props, payload = Payload}, + State = #state{owner = Owner, request_handler = RequestHandler}) -> + case RequestHandler of + ?NO_HANDLER -> + Owner ! {publish, #{qos => QoS, dup => Dup, retain => Retain, packet_id => PacketId, + topic => Topic, properties => Props, payload => Payload, + client_pid => self()}}; + _ -> + ok + end, + State. + +packet_to_msg(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + dup = Dup, + qos = QoS, + retain = R}, + variable = #mqtt_packet_publish{topic_name = Topic, + packet_id = PacketId, + properties = Props}, + payload = Payload}) -> + #mqtt_msg{qos = QoS, retain = R, dup = Dup, packet_id = PacketId, + topic = Topic, props = Props, payload = Payload}. + +msg_to_packet(#mqtt_msg{qos = QoS, dup = Dup, retain = Retain, packet_id = PacketId, + topic = Topic, props = Props, payload = Payload}) -> + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + qos = QoS, + retain = Retain, + dup = Dup}, + variable = #mqtt_packet_publish{topic_name = Topic, + packet_id = PacketId, + properties = Props}, + payload = Payload}. + +%%------------------------------------------------------------------------------ +%% Socket Connect/Send + +sock_connect(Hosts, SockOpts, Timeout) -> + sock_connect(Hosts, SockOpts, Timeout, {error, no_hosts}). + +sock_connect([], _SockOpts, _Timeout, LastErr) -> + LastErr; +sock_connect([{Host, Port} | Hosts], SockOpts, Timeout, _LastErr) -> + case emqx_client_sock:connect(Host, Port, SockOpts, Timeout) of + {ok, Socket} -> {ok, Socket}; + Err = {error, _Reason} -> + sock_connect(Hosts, SockOpts, Timeout, Err) + end. + +hosts(#state{hosts = [], host = Host, port = Port}) -> + [{Host, Port}]; +hosts(#state{hosts = Hosts}) -> Hosts. + +send_puback(Packet, State) -> + case send(Packet, State) of + {ok, NewState} -> {keep_state, NewState}; + {error, Reason} -> {stop, Reason} + end. + +send(Msg, State) when is_record(Msg, mqtt_msg) -> + send(msg_to_packet(Msg), State); + +send(Packet, State = #state{socket = Sock, proto_ver = Ver}) + when is_record(Packet, mqtt_packet) -> + Data = emqx_frame:serialize(Packet, #{version => Ver}), + emqx_logger:debug("SEND Data: ~p", [Data]), + case emqx_client_sock:send(Sock, Data) of + ok -> {ok, next_packet_id(State)}; + Error -> Error + end. + +run_sock(State = #state{socket = Sock}) -> + emqx_client_sock:setopts(Sock, [{active, once}]), State. + +%%------------------------------------------------------------------------------ +%% Receive Loop + +receive_loop(<<>>, State) -> + {keep_state, State}; + +receive_loop(Bytes, State = #state{parse_state = ParseState}) -> + case catch emqx_frame:parse(Bytes, ParseState) of + {ok, Packet, Rest} -> + ok = gen_statem:cast(self(), Packet), + receive_loop(Rest, init_parse_state(State)); + {more, NewParseState} -> + {keep_state, State#state{parse_state = NewParseState}}; + {error, Reason} -> + {stop, Reason}; + {'EXIT', Error} -> + {stop, Error} + end. + +%%------------------------------------------------------------------------------ +%% Next packet id + +next_packet_id(State = #state{last_packet_id = 16#ffff}) -> + State#state{last_packet_id = 1}; + +next_packet_id(State = #state{last_packet_id = Id}) -> + State#state{last_packet_id = Id + 1}. diff --git a/src/emqx_client_sock.erl b/src/emqx_client_sock.erl new file mode 100644 index 000000000..505454e2d --- /dev/null +++ b/src/emqx_client_sock.erl @@ -0,0 +1,101 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_client_sock). + +-export([connect/4, send/2, close/1]). + +-export([sockname/1, setopts/2, getstat/2]). + +-record(ssl_socket, {tcp, ssl}). + +-type(socket() :: inet:socket() | #ssl_socket{}). + +-type(sockname() :: {inet:ip_address(), inet:port_number()}). + +-type(option() :: gen_tcp:connect_option() | {ssl_opts, [ssl:ssl_option()]}). + +-export_type([socket/0, option/0]). + +-define(DEFAULT_TCP_OPTIONS, [binary, {packet, raw}, {active, false}, + {nodelay, true}, {reuseaddr, true}]). + +-spec(connect(inet:ip_address() | inet:hostname(), + inet:port_number(), [option()], timeout()) + -> {ok, socket()} | {error, term()}). +connect(Host, Port, SockOpts, Timeout) -> + TcpOpts = emqx_misc:merge_opts(?DEFAULT_TCP_OPTIONS, + lists:keydelete(ssl_opts, 1, SockOpts)), + case gen_tcp:connect(Host, Port, TcpOpts, Timeout) of + {ok, Sock} -> + case lists:keyfind(ssl_opts, 1, SockOpts) of + {ssl_opts, SslOpts} -> + ssl_upgrade(Sock, SslOpts, Timeout); + false -> {ok, Sock} + end; + {error, Reason} -> + {error, Reason} + end. + +ssl_upgrade(Sock, SslOpts, Timeout) -> + TlsVersions = proplists:get_value(versions, SslOpts, []), + Ciphers = proplists:get_value(ciphers, SslOpts, default_ciphers(TlsVersions)), + SslOpts2 = emqx_misc:merge_opts(SslOpts, [{ciphers, Ciphers}]), + case ssl:connect(Sock, SslOpts2, Timeout) of + {ok, SslSock} -> + ok = ssl:controlling_process(SslSock, self()), + {ok, #ssl_socket{tcp = Sock, ssl = SslSock}}; + {error, Reason} -> {error, Reason} + end. + +-spec(send(socket(), iodata()) -> ok | {error, einval | closed}). +send(Sock, Data) when is_port(Sock) -> + try erlang:port_command(Sock, Data) of + true -> ok + catch + error:badarg -> {error, einval} + end; +send(#ssl_socket{ssl = SslSock}, Data) -> + ssl:send(SslSock, Data). + +-spec(close(socket()) -> ok). +close(Sock) when is_port(Sock) -> + gen_tcp:close(Sock); +close(#ssl_socket{ssl = SslSock}) -> + ssl:close(SslSock). + +-spec(setopts(socket(), [gen_tcp:option() | ssl:socketoption()]) -> ok). +setopts(Sock, Opts) when is_port(Sock) -> + inet:setopts(Sock, Opts); +setopts(#ssl_socket{ssl = SslSock}, Opts) -> + ssl:setopts(SslSock, Opts). + +-spec(getstat(socket(), [atom()]) + -> {ok, [{atom(), integer()}]} | {error, term()}). +getstat(Sock, Options) when is_port(Sock) -> + inet:getstat(Sock, Options); +getstat(#ssl_socket{tcp = Sock}, Options) -> + inet:getstat(Sock, Options). + +-spec(sockname(socket()) -> {ok, sockname()} | {error, term()}). +sockname(Sock) when is_port(Sock) -> + inet:sockname(Sock); +sockname(#ssl_socket{ssl = SslSock}) -> + ssl:sockname(SslSock). + +default_ciphers(TlsVersions) -> + lists:foldl( + fun(TlsVer, Ciphers) -> + Ciphers ++ ssl:cipher_suites(all, TlsVer) + end, [], TlsVersions). diff --git a/src/emqx_cm.erl b/src/emqx_cm.erl new file mode 100644 index 000000000..b8a57bc50 --- /dev/null +++ b/src/emqx_cm.erl @@ -0,0 +1,191 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_cm). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/0]). + +-export([register_connection/1, register_connection/2]). +-export([unregister_connection/1, unregister_connection/2]). +-export([get_conn_attrs/1, get_conn_attrs/2]). +-export([set_conn_attrs/2, set_conn_attrs/3]). +-export([get_conn_stats/1, get_conn_stats/2]). +-export([set_conn_stats/2, set_conn_stats/3]). +-export([lookup_conn_pid/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +%% internal export +-export([stats_fun/0]). + +-define(CM, ?MODULE). + +%% ETS tables for connection management. +-define(CONN_TAB, emqx_conn). +-define(CONN_ATTRS_TAB, emqx_conn_attrs). +-define(CONN_STATS_TAB, emqx_conn_stats). + +-define(BATCH_SIZE, 100000). + +%% @doc Start the connection manager. +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?CM}, ?MODULE, [], []). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% @doc Register a connection. +-spec(register_connection(emqx_types:client_id()) -> ok). +register_connection(ClientId) when is_binary(ClientId) -> + register_connection(ClientId, self()). + +-spec(register_connection(emqx_types:client_id(), pid()) -> ok). +register_connection(ClientId, ConnPid) when is_binary(ClientId), is_pid(ConnPid) -> + true = ets:insert(?CONN_TAB, {ClientId, ConnPid}), + notify({registered, ClientId, ConnPid}). + +%% @doc Unregister a connection. +-spec(unregister_connection(emqx_types:client_id()) -> ok). +unregister_connection(ClientId) when is_binary(ClientId) -> + unregister_connection(ClientId, self()). + +-spec(unregister_connection(emqx_types:client_id(), pid()) -> ok). +unregister_connection(ClientId, ConnPid) when is_binary(ClientId), is_pid(ConnPid) -> + true = do_unregister_connection({ClientId, ConnPid}), + notify({unregistered, ConnPid}). + +do_unregister_connection(Conn) -> + true = ets:delete(?CONN_STATS_TAB, Conn), + true = ets:delete(?CONN_ATTRS_TAB, Conn), + true = ets:delete_object(?CONN_TAB, Conn). + +%% @doc Get conn attrs +-spec(get_conn_attrs(emqx_types:client_id()) -> list()). +get_conn_attrs(ClientId) when is_binary(ClientId) -> + ConnPid = lookup_conn_pid(ClientId), + get_conn_attrs(ClientId, ConnPid). + +-spec(get_conn_attrs(emqx_types:client_id(), pid()) -> list()). +get_conn_attrs(ClientId, ConnPid) when is_binary(ClientId) -> + emqx_tables:lookup_value(?CONN_ATTRS_TAB, {ClientId, ConnPid}, []). + +%% @doc Set conn attrs +-spec(set_conn_attrs(emqx_types:client_id(), list()) -> true). +set_conn_attrs(ClientId, Attrs) when is_binary(ClientId) -> + set_conn_attrs(ClientId, self(), Attrs). + +-spec(set_conn_attrs(emqx_types:client_id(), pid(), list()) -> true). +set_conn_attrs(ClientId, ConnPid, Attrs) when is_binary(ClientId), is_pid(ConnPid) -> + Conn = {ClientId, ConnPid}, + ets:insert(?CONN_ATTRS_TAB, {Conn, Attrs}). + +%% @doc Get conn stats +-spec(get_conn_stats(emqx_types:client_id()) -> list(emqx_stats:stats())). +get_conn_stats(ClientId) when is_binary(ClientId) -> + ConnPid = lookup_conn_pid(ClientId), + get_conn_stats(ClientId, ConnPid). + +-spec(get_conn_stats(emqx_types:client_id(), pid()) -> list(emqx_stats:stats())). +get_conn_stats(ClientId, ConnPid) when is_binary(ClientId) -> + Conn = {ClientId, ConnPid}, + emqx_tables:lookup_value(?CONN_STATS_TAB, Conn, []). + +%% @doc Set conn stats. +-spec(set_conn_stats(emqx_types:client_id(), list(emqx_stats:stats())) -> true). +set_conn_stats(ClientId, Stats) when is_binary(ClientId) -> + set_conn_stats(ClientId, self(), Stats). + +-spec(set_conn_stats(emqx_types:client_id(), pid(), list(emqx_stats:stats())) -> true). +set_conn_stats(ClientId, ConnPid, Stats) when is_binary(ClientId), is_pid(ConnPid) -> + Conn = {ClientId, ConnPid}, + ets:insert(?CONN_STATS_TAB, {Conn, Stats}). + +%% @doc Lookup connection pid. +-spec(lookup_conn_pid(emqx_types:client_id()) -> pid() | undefined). +lookup_conn_pid(ClientId) when is_binary(ClientId) -> + emqx_tables:lookup_value(?CONN_TAB, ClientId). + +notify(Msg) -> + gen_server:cast(?CM, {notify, Msg}). + +%%----------------------------------------------------------------------------- +%% gen_server callbacks +%%----------------------------------------------------------------------------- + +init([]) -> + TabOpts = [public, set, {write_concurrency, true}], + ok = emqx_tables:new(?CONN_TAB, [{read_concurrency, true} | TabOpts]), + ok = emqx_tables:new(?CONN_ATTRS_TAB, TabOpts), + ok = emqx_tables:new(?CONN_STATS_TAB, TabOpts), + ok = emqx_stats:update_interval(conn_stats, fun ?MODULE:stats_fun/0), + {ok, #{conn_pmon => emqx_pmon:new()}}. + +handle_call(Req, _From, State) -> + emqx_logger:error("[CM] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({notify, {registered, ClientId, ConnPid}}, State = #{conn_pmon := PMon}) -> + {noreply, State#{conn_pmon := emqx_pmon:monitor(ConnPid, ClientId, PMon)}}; + +handle_cast({notify, {unregistered, ConnPid}}, State = #{conn_pmon := PMon}) -> + {noreply, State#{conn_pmon := emqx_pmon:demonitor(ConnPid, PMon)}}; + +handle_cast(Msg, State) -> + emqx_logger:error("[CM] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{conn_pmon := PMon}) -> + ConnPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + {Items, PMon1} = emqx_pmon:erase_all(ConnPids, PMon), + ok = emqx_pool:async_submit( + fun lists:foreach/2, [fun clean_down/1, Items]), + {noreply, State#{conn_pmon := PMon1}}; + +handle_info(Info, State) -> + emqx_logger:error("[CM] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + emqx_stats:cancel_update(conn_stats). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +clean_down({Pid, ClientId}) -> + Conn = {ClientId, Pid}, + case ets:member(?CONN_TAB, ClientId) + orelse ets:member(?CONN_ATTRS_TAB, Conn) of + true -> + do_unregister_connection(Conn); + false -> false + end. + +stats_fun() -> + case ets:info(?CONN_TAB, size) of + undefined -> ok; + Size -> emqx_stats:setstat('connections/count', 'connections/max', Size) + end. + diff --git a/src/emqttd_sysmon_sup.erl b/src/emqx_cm_sup.erl similarity index 54% rename from src/emqttd_sysmon_sup.erl rename to src/emqx_cm_sup.erl index 884112a00..e8c99c16b 100644 --- a/src/emqttd_sysmon_sup.erl +++ b/src/emqx_cm_sup.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,24 +11,30 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_sysmon_sup). +-module(emqx_cm_sup). -behaviour(supervisor). -%% API -export([start_link/0]). -%% Supervisor callbacks -export([init/1]). start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - {ok, Env} = emqttd:env(sysmon), - Sysmon = {sysmon, {emqttd_sysmon, start_link, [Env]}, - permanent, 5000, worker, [emqttd_sysmon]}, - {ok, {{one_for_one, 10, 100}, [Sysmon]}}. + Banned = #{id => banned, + start => {emqx_banned, start_link, []}, + restart => permanent, + shutdown => 1000, + type => worker, + modules => [emqx_banned]}, + Manager = #{id => manager, + start => {emqx_cm, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_cm]}, + {ok, {{one_for_one, 10, 100}, [Banned, Manager]}}. diff --git a/src/emqx_config.erl b/src/emqx_config.erl new file mode 100644 index 000000000..bab3eab56 --- /dev/null +++ b/src/emqx_config.erl @@ -0,0 +1,132 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc Hot Configuration +%% +%% TODO: How to persist the configuration? +%% +%% 1. Store in mnesia database? +%% 2. Store in dets? +%% 3. Store in data/app.config? +%% + +-module(emqx_config). + +-export([get_env/1, get_env/2]). + +-export([populate/1]). + +-export([read/1, write/2, dump/2, reload/1, get/2, get/3, set/3]). + +-type(env() :: {atom(), term()}). + +-define(APP, emqx). + +%% @doc Get environment +-spec(get_env(Key :: atom()) -> term() | undefined). +get_env(Key) -> + get_env(Key, undefined). + +-spec(get_env(Key :: atom(), Default :: term()) -> term()). +get_env(Key, Default) -> + application:get_env(?APP, Key, Default). + +%% TODO: +populate(_App) -> + ok. + +%% @doc Read the configuration of an application. +-spec(read(atom()) -> {ok, list(env())} | {error, term()}). +read(App) -> + %% TODO: + %% 1. Read the app.conf from etc folder + %% 2. Cuttlefish to read the conf + %% 3. Return the terms and schema + % {error, unsupported}. + {ok, read_(App)}. + +%% @doc Reload configuration of an application. +-spec(reload(atom()) -> ok | {error, term()}). +reload(_App) -> + %% TODO + %% 1. Read the app.conf from etc folder + %% 2. Cuttlefish to generate config terms. + %% 3. set/3 to apply the config + ok. + +-spec(write(atom(), list(env())) -> ok | {error, term()}). +write(_App, _Terms) -> ok. + % Configs = lists:map(fun({Key, Val}) -> + % {cuttlefish_variable:tokenize(binary_to_list(Key)), binary_to_list(Val)} + % end, Terms), + % Path = lists:concat([code:priv_dir(App), "/", App, ".schema"]), + % Schema = cuttlefish_schema:files([Path]), + % case cuttlefish_generator:map(Schema, Configs) of + % [{App, Configs1}] -> + % emqx_cli_config:write_config(App, Configs), + % lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Configs1); + % _ -> + % error + % end. + +-spec(dump(atom(), list(env())) -> ok | {error, term()}). +dump(_App, _Terms) -> + %% TODO + ok. + +-spec(set(atom(), list(), list()) -> ok). +set(_App, _Par, _Val) -> ok. + % emqx_cli_config:run(["config", + % "set", + % lists:concat([Par, "=", Val]), + % lists:concat(["--app=", App])]). + +-spec(get(atom(), list()) -> undefined | {ok, term()}). +get(_App, _Par) -> error(no_impl). + % case emqx_cli_config:get_cfg(App, Par) of + % undefined -> undefined; + % Val -> {ok, Val} + % end. + +-spec(get(atom(), list(), atom()) -> term()). +get(_App, _Par, _Def) -> error(no_impl). + % emqx_cli_config:get_cfg(App, Par, Def). + + +read_(_App) -> error(no_impl). + % Configs = emqx_cli_config:read_config(App), + % Path = lists:concat([code:priv_dir(App), "/", App, ".schema"]), + % case filelib:is_file(Path) of + % false -> + % []; + % true -> + % {_, Mappings, _} = cuttlefish_schema:files([Path]), + % OptionalCfg = lists:foldl(fun(Map, Acc) -> + % Key = cuttlefish_mapping:variable(Map), + % case proplists:get_value(Key, Configs) of + % undefined -> + % [{cuttlefish_variable:format(Key), "", cuttlefish_mapping:doc(Map), false} | Acc]; + % _ -> Acc + % end + % end, [], Mappings), + % RequiredCfg = lists:foldl(fun({Key, Val}, Acc) -> + % case lists:keyfind(Key, 2, Mappings) of + % false -> Acc; + % Map -> + % [{cuttlefish_variable:format(Key), Val, cuttlefish_mapping:doc(Map), true} | Acc] + % end + % end, [], Configs), + % RequiredCfg ++ OptionalCfg + % end. + diff --git a/src/emqx_connection.erl b/src/emqx_connection.erl new file mode 100644 index 000000000..9f2572b32 --- /dev/null +++ b/src/emqx_connection.erl @@ -0,0 +1,443 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_connection). + +-behaviour(gen_server). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-define(LOG_HEADER, "[MQTT]"). +-include("logger.hrl"). + +-export([start_link/3]). +-export([info/1, attrs/1, stats/1]). +-export([kick/1]). +-export([session/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-record(state, { + transport, + socket, + peername, + sockname, + conn_state, + active_n, + proto_state, + parser_state, + gc_state, + keepalive, + enable_stats, + stats_timer, + rate_limit, + pub_limit, + limit_timer, + idle_timeout + }). + +-define(DEFAULT_ACTIVE_N, 100). +-define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). + +start_link(Transport, Socket, Options) -> + {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Socket, Options]])}. + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% for debug +info(CPid) when is_pid(CPid) -> + call(CPid, info); + +info(#state{transport = Transport, + socket = Socket, + peername = Peername, + sockname = Sockname, + conn_state = ConnState, + active_n = ActiveN, + rate_limit = RateLimit, + pub_limit = PubLimit, + proto_state = ProtoState}) -> + ConnInfo = [{socktype, Transport:type(Socket)}, + {peername, Peername}, + {sockname, Sockname}, + {conn_state, ConnState}, + {active_n, ActiveN}, + {rate_limit, esockd_rate_limit:info(RateLimit)}, + {pub_limit, esockd_rate_limit:info(PubLimit)}], + ProtoInfo = emqx_protocol:info(ProtoState), + lists:usort(lists:append(ConnInfo, ProtoInfo)). + +%% for dashboard +attrs(CPid) when is_pid(CPid) -> + call(CPid, attrs); + +attrs(#state{peername = Peername, + sockname = Sockname, + proto_state = ProtoState}) -> + SockAttrs = [{peername, Peername}, + {sockname, Sockname}], + ProtoAttrs = emqx_protocol:attrs(ProtoState), + lists:usort(lists:append(SockAttrs, ProtoAttrs)). + +%% Conn stats +stats(CPid) when is_pid(CPid) -> + call(CPid, stats); + +stats(#state{transport = Transport, + socket = Socket, + proto_state = ProtoState}) -> + lists:append([emqx_misc:proc_stats(), + emqx_protocol:stats(ProtoState), + case Transport:getstat(Socket, ?SOCK_STATS) of + {ok, Ss} -> Ss; + {error, _} -> [] + end]). + +kick(CPid) -> call(CPid, kick). + +session(CPid) -> call(CPid, session). + +call(CPid, Req) -> + gen_server:call(CPid, Req, infinity). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Transport, RawSocket, Options]) -> + case Transport:wait(RawSocket) of + {ok, Socket} -> + Zone = proplists:get_value(zone, Options), + {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]), + {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]), + Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]), + RateLimit = init_limiter(proplists:get_value(rate_limit, Options)), + PubLimit = init_limiter(emqx_zone:get_env(Zone, publish_limit)), + ActiveN = proplists:get_value(active_n, Options, ?DEFAULT_ACTIVE_N), + EnableStats = emqx_zone:get_env(Zone, enable_stats, true), + IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000), + SendFun = send_fun(Transport, Socket), + ProtoState = emqx_protocol:init(#{peername => Peername, + sockname => Sockname, + peercert => Peercert, + sendfun => SendFun}, Options), + ParserState = emqx_protocol:parser(ProtoState), + GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), + GcState = emqx_gc:init(GcPolicy), + State = run_socket(#state{transport = Transport, + socket = Socket, + peername = Peername, + conn_state = running, + active_n = ActiveN, + rate_limit = RateLimit, + pub_limit = PubLimit, + proto_state = ProtoState, + parser_state = ParserState, + gc_state = GcState, + enable_stats = EnableStats, + idle_timeout = IdleTimout + }), + ok = emqx_misc:init_proc_mng_policy(Zone), + emqx_logger:set_metadata_peername(esockd_net:format(Peername)), + gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], + State, self(), IdleTimout); + {error, Reason} -> + {stop, Reason} + end. + +init_limiter(undefined) -> + undefined; +init_limiter({Rate, Burst}) -> + esockd_rate_limit:new(Rate, Burst). + +send_fun(Transport, Socket) -> + fun(Packet, Options) -> + Data = emqx_frame:serialize(Packet, Options), + try Transport:async_send(Socket, Data) of + ok -> + emqx_metrics:trans(inc, 'bytes/sent', iolist_size(Data)), + ok; + Error -> Error + catch + error:Error -> + {error, Error} + end + end. + +handle_call(info, _From, State) -> + {reply, info(State), State}; + +handle_call(attrs, _From, State) -> + {reply, attrs(State), State}; + +handle_call(stats, _From, State) -> + {reply, stats(State), State}; + +handle_call(kick, _From, State) -> + {stop, {shutdown, kicked}, ok, State}; + +handle_call(session, _From, State = #state{proto_state = ProtoState}) -> + {reply, emqx_protocol:session(ProtoState), State}; + +handle_call(Req, _From, State) -> + ?LOG(error, "unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?LOG(error, "unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) -> + case emqx_protocol:deliver(PubOrAck, ProtoState) of + {ok, ProtoState1} -> + State1 = State#state{proto_state = ProtoState1}, + {noreply, maybe_gc(PubOrAck, ensure_stats_timer(State1))}; + {error, Reason} -> + shutdown(Reason, State) + end; + +handle_info({timeout, Timer, emit_stats}, + State = #state{stats_timer = Timer, + proto_state = ProtoState, + gc_state = GcState}) -> + emqx_metrics:commit(), + emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)), + NewState = State#state{stats_timer = undefined}, + Limits = erlang:get(force_shutdown_policy), + case emqx_misc:conn_proc_mng_policy(Limits) of + continue -> + {noreply, NewState}; + hibernate -> + %% going to hibernate, reset gc stats + GcState1 = emqx_gc:reset(GcState), + {noreply, NewState#state{gc_state = GcState1}, hibernate}; + {shutdown, Reason} -> + ?LOG(warning, "shutdown due to ~p", [Reason]), + shutdown(Reason, NewState) + end; + +handle_info(timeout, State) -> + shutdown(idle_timeout, State); + +handle_info({shutdown, Reason}, State) -> + shutdown(Reason, State); + +handle_info({shutdown, discard, {ClientId, ByPid}}, State) -> + ?LOG(warning, "discarded by ~s:~p", [ClientId, ByPid]), + shutdown(discard, State); + +handle_info({shutdown, conflict, {ClientId, NewPid}}, State) -> + ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]), + shutdown(conflict, State); + +handle_info({TcpOrSsL, _Sock, Data}, State) when TcpOrSsL =:= tcp; TcpOrSsL =:= ssl -> + process_incoming(Data, State); + +%% Rate limit here, cool:) +handle_info({tcp_passive, _Sock}, State) -> + {noreply, run_socket(ensure_rate_limit(State))}; +%% FIXME Later +handle_info({ssl_passive, _Sock}, State) -> + {noreply, run_socket(ensure_rate_limit(State))}; + +handle_info({Err, _Sock, Reason}, State) when Err =:= tcp_error; Err =:= ssl_error -> + shutdown(Reason, State); + +handle_info({Closed, _Sock}, State) when Closed =:= tcp_closed; Closed =:= ssl_closed -> + shutdown(closed, State); + +%% Rate limit timer +handle_info(activate_sock, State) -> + {noreply, run_socket(State#state{conn_state = running, limit_timer = undefined})}; + +handle_info({inet_reply, _Sock, ok}, State) -> + {noreply, State}; + +handle_info({inet_reply, _Sock, {error, Reason}}, State) -> + shutdown(Reason, State); + +handle_info({keepalive, start, Interval}, State = #state{transport = Transport, socket = Socket}) -> + ?LOG(debug, "Keepalive at the interval of ~p", [Interval]), + StatFun = fun() -> + case Transport:getstat(Socket, [recv_oct]) of + {ok, [{recv_oct, RecvOct}]} -> {ok, RecvOct}; + Error -> Error + end + end, + case emqx_keepalive:start(StatFun, Interval, {keepalive, check}) of + {ok, KeepAlive} -> + {noreply, State#state{keepalive = KeepAlive}}; + {error, Error} -> + shutdown(Error, State) + end; + +handle_info({keepalive, check}, State = #state{keepalive = KeepAlive}) -> + case emqx_keepalive:check(KeepAlive) of + {ok, KeepAlive1} -> + {noreply, State#state{keepalive = KeepAlive1}}; + {error, timeout} -> + shutdown(keepalive_timeout, State); + {error, Error} -> + shutdown(Error, State) + end; + +handle_info(Info, State) -> + ?LOG(error, "unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(Reason, #state{transport = Transport, + socket = Socket, + keepalive = KeepAlive, + proto_state = ProtoState}) -> + ?LOG(debug, "Terminated for ~p", [Reason]), + Transport:fast_close(Socket), + emqx_keepalive:cancel(KeepAlive), + case {ProtoState, Reason} of + {undefined, _} -> ok; + {_, {shutdown, Error}} -> + emqx_protocol:shutdown(Error, ProtoState); + {_, Reason} -> + emqx_protocol:shutdown(Reason, ProtoState) + end. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internals: process incoming, parse and handle packets +%%------------------------------------------------------------------------------ + +process_incoming(Data, State) -> + Oct = iolist_size(Data), + ?LOG(debug, "RECV ~p", [Data]), + emqx_pd:update_counter(incoming_bytes, Oct), + emqx_metrics:trans(inc, 'bytes/received', Oct), + case handle_packet(Data, State) of + {noreply, State1} -> + State2 = maybe_gc({1, Oct}, State1), + {noreply, ensure_stats_timer(State2)}; + Shutdown -> Shutdown + end. + +%% Parse and handle packets +handle_packet(<<>>, State) -> + {noreply, State}; + +handle_packet(Data, State = #state{proto_state = ProtoState, + parser_state = ParserState, + idle_timeout = IdleTimeout}) -> + try emqx_frame:parse(Data, ParserState) of + {more, ParserState1} -> + {noreply, State#state{parser_state = ParserState1}, IdleTimeout}; + {ok, Packet = ?PACKET(Type), Rest} -> + emqx_metrics:received(Packet), + (Type == ?PUBLISH) andalso emqx_pd:update_counter(incoming_pubs, 1), + case emqx_protocol:received(Packet, ProtoState) of + {ok, ProtoState1} -> + handle_packet(Rest, reset_parser(State#state{proto_state = ProtoState1})); + {error, Reason} -> + ?LOG(error, "Process packet error - ~p", [Reason]), + shutdown(Reason, State); + {error, Reason, ProtoState1} -> + shutdown(Reason, State#state{proto_state = ProtoState1}); + {stop, Error, ProtoState1} -> + stop(Error, State#state{proto_state = ProtoState1}) + end; + {error, Reason} -> + ?LOG(error, "Parse frame error - ~p", [Reason]), + shutdown(Reason, State) + catch + _:Error -> + ?LOG(error, "Parse failed for ~p~nError data:~p", [Error, Data]), + shutdown(parse_error, State) + end. + +reset_parser(State = #state{proto_state = ProtoState}) -> + State#state{parser_state = emqx_protocol:parser(ProtoState)}. + +%%------------------------------------------------------------------------------ +%% Ensure rate limit + +ensure_rate_limit(State = #state{rate_limit = Rl, pub_limit = Pl}) -> + Limiters = [{Pl, #state.pub_limit, emqx_pd:reset_counter(incoming_pubs)}, + {Rl, #state.rate_limit, emqx_pd:reset_counter(incoming_bytes)}], + ensure_rate_limit(Limiters, State). + +ensure_rate_limit([], State) -> + State; +ensure_rate_limit([{undefined, _Pos, _Cnt}|Limiters], State) -> + ensure_rate_limit(Limiters, State); +ensure_rate_limit([{Rl, Pos, Cnt}|Limiters], State) -> + case esockd_rate_limit:check(Cnt, Rl) of + {0, Rl1} -> + ensure_rate_limit(Limiters, setelement(Pos, State, Rl1)); + {Pause, Rl1} -> + TRef = erlang:send_after(Pause, self(), activate_sock), + setelement(Pos, State#state{conn_state = blocked, limit_timer = TRef}, Rl1) + end. + +%%------------------------------------------------------------------------------ +%% Activate socket + +run_socket(State = #state{conn_state = blocked}) -> + State; + +run_socket(State = #state{transport = Transport, socket = Socket, active_n = N}) -> + TrueOrN = case Transport:is_ssl(Socket) of + true -> true; %% Cannot set '{active, N}' for SSL:( + false -> N + end, + ensure_ok_or_exit(Transport:setopts(Socket, [{active, TrueOrN}])), + State. + +ensure_ok_or_exit(ok) -> ok; +ensure_ok_or_exit({error, Reason}) -> + self() ! {shutdown, Reason}. + +%%------------------------------------------------------------------------------ +%% Ensure stats timer + +ensure_stats_timer(State = #state{enable_stats = true, + stats_timer = undefined, + idle_timeout = IdleTimeout}) -> + State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; +ensure_stats_timer(State) -> State. + +%%------------------------------------------------------------------------------ +%% Maybe GC + +maybe_gc(_, State = #state{gc_state = undefined}) -> + State; +maybe_gc({publish, _PacketId, #message{payload = Payload}}, State) -> + Oct = iolist_size(Payload), + maybe_gc({1, Oct}, State); +maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> + {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt), + State#state{gc_state = GCSt1}; +maybe_gc(_, State) -> + State. + +%%------------------------------------------------------------------------------ +%% Shutdown or stop + +shutdown(Reason, State) -> + stop({shutdown, Reason}, State). + +stop(Reason, State) -> + {stop, Reason, State}. diff --git a/src/emqx_ctl.erl b/src/emqx_ctl.erl new file mode 100644 index 000000000..1d2fb13a3 --- /dev/null +++ b/src/emqx_ctl.erl @@ -0,0 +1,153 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_ctl). + +-behaviour(gen_server). + +-export([start_link/0]). +-export([register_command/2, register_command/3, unregister_command/1]). +-export([run_command/1, run_command/2, lookup_command/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-record(state, {seq = 0}). + +-type(cmd() :: atom()). + +-define(SERVER, ?MODULE). +-define(TAB, emqx_command). + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +-spec(register_command(cmd(), {module(), atom()}) -> ok). +register_command(Cmd, MF) when is_atom(Cmd) -> + register_command(Cmd, MF, []). + +-spec(register_command(cmd(), {module(), atom()}, list()) -> ok). +register_command(Cmd, MF, Opts) when is_atom(Cmd) -> + cast({register_command, Cmd, MF, Opts}). + +-spec(unregister_command(cmd()) -> ok). +unregister_command(Cmd) when is_atom(Cmd) -> + cast({unregister_command, Cmd}). + +cast(Msg) -> + gen_server:cast(?SERVER, Msg). + +run_command([]) -> + run_command(help, []); +run_command([Cmd | Args]) -> + run_command(list_to_atom(Cmd), Args). + +-spec(run_command(cmd(), [string()]) -> ok | {error, term()}). +run_command(help, []) -> + usage(); +run_command(Cmd, Args) when is_atom(Cmd) -> + case lookup_command(Cmd) of + [{Mod, Fun}] -> + try Mod:Fun(Args) of + _ -> ok + catch + _:Reason:Stacktrace -> + emqx_logger:error("[CTL] CMD Error:~p, Stacktrace:~p", [Reason, Stacktrace]), + {error, Reason} + end; + [] -> + usage(), {error, cmd_not_found} + end. + +-spec(lookup_command(cmd()) -> [{module(), atom()}]). +lookup_command(Cmd) when is_atom(Cmd) -> + case ets:match(?TAB, {{'_', Cmd}, '$1', '_'}) of + [El] -> El; + [] -> [] + end. + +usage() -> + io:format("Usage: ~s~n", [?MODULE]), + [begin io:format("~80..-s~n", [""]), Mod:Cmd(usage) end + || {_, {Mod, Cmd}, _} <- ets:tab2list(?TAB)]. + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + ok = emqx_tables:new(?TAB, [protected, ordered_set]), + {ok, #state{seq = 0}}. + +handle_call(Req, _From, State) -> + emqx_logger:error("unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({register_command, Cmd, MF, Opts}, State = #state{seq = Seq}) -> + case ets:match(?TAB, {{'$1', Cmd}, '_', '_'}) of + [] -> ets:insert(?TAB, {{Seq, Cmd}, MF, Opts}); + [[OriginSeq] | _] -> + emqx_logger:warning("[CTL] cmd ~s is overidden by ~p", [Cmd, MF]), + ets:insert(?TAB, {{OriginSeq, Cmd}, MF, Opts}) + end, + noreply(next_seq(State)); + +handle_cast({unregister_command, Cmd}, State) -> + ets:match_delete(?TAB, {{'_', Cmd}, '_', '_'}), + noreply(State); + +handle_cast(Msg, State) -> + emqx_logger:error("Unexpected cast: ~p", [Msg]), + noreply(State). + +handle_info(Info, State) -> + emqx_logger:error("unexpected info: ~p", [Info]), + noreply(State). + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal Function +%%------------------------------------------------------------------------------ + +noreply(State) -> + {noreply, State, hibernate}. + +next_seq(State = #state{seq = Seq}) -> + State#state{seq = Seq + 1}. + +-ifdef(TEST). + +-include_lib("eunit/include/eunit.hrl"). + +register_command_test_() -> + {setup, + fun() -> + {ok, InitState} = emqx_ctl:init([]), + InitState + end, + fun(State) -> + ok = emqx_ctl:terminate(shutdown, State) + end, + fun(State = #state{seq = Seq}) -> + emqx_ctl:handle_cast({register_command, test0, {?MODULE, test0}, []}, State), + [?_assertMatch([{{0,test0},{?MODULE, test0}, []}], ets:lookup(?TAB, {Seq,test0}))] + end + }. + +-endif. diff --git a/src/emqx_flapping.erl b/src/emqx_flapping.erl new file mode 100644 index 000000000..c1cefd893 --- /dev/null +++ b/src/emqx_flapping.erl @@ -0,0 +1,73 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc TODO: +%% 1. Flapping Detection +%% 2. Conflict Detection? +-module(emqx_flapping). + +%% Use ets:update_counter??? + +-behaviour(gen_server). + +-export([start_link/0]). + +-export([is_banned/1, banned/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, {}). + +-spec(start_link() -> {ok, pid()} | ignore | {error, any()}). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +is_banned(ClientId) -> + ets:member(banned, ClientId). + +banned(ClientId) -> + ets:insert(banned, {ClientId, os:timestamp()}). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + %% ets:new(banned, [public, ordered_set, named_table]), + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + + diff --git a/src/emqx_frame.erl b/src/emqx_frame.erl new file mode 100644 index 000000000..075f0a11e --- /dev/null +++ b/src/emqx_frame.erl @@ -0,0 +1,636 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_frame). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-export([initial_state/0, initial_state/1]). +-export([parse/2]). +-export([serialize/1, serialize/2]). + +-type(options() :: #{max_packet_size => 1..?MAX_PACKET_SIZE, + version => emqx_mqtt_types:version()}). + +-type(parse_state() :: {none, options()} | cont_fun(binary())). + +-type(cont_fun(Bin) :: fun((Bin) -> {ok, emqx_mqtt_types:packet(), binary()} + | {more, cont_fun(Bin)})). + +-export_type([options/0, parse_state/0]). + +-define(DEFAULT_OPTIONS, #{max_packet_size => ?MAX_PACKET_SIZE, + version => ?MQTT_PROTO_V4}). + +%%------------------------------------------------------------------------------ +%% Init parse state +%%------------------------------------------------------------------------------ + +-spec(initial_state() -> {none, options()}). +initial_state() -> + initial_state(#{}). + +-spec(initial_state(options()) -> {none, options()}). +initial_state(Options) when is_map(Options) -> + {none, merge_opts(Options)}. + +merge_opts(Options) -> + maps:merge(?DEFAULT_OPTIONS, Options). + +%%------------------------------------------------------------------------------ +%% Parse MQTT Frame +%%------------------------------------------------------------------------------ + +-spec(parse(binary(), parse_state()) -> {ok, emqx_mqtt_types:packet(), binary()} | + {more, cont_fun(binary())}). +parse(<<>>, {none, Options}) -> + {more, fun(Bin) -> parse(Bin, {none, Options}) end}; +parse(<>, {none, Options}) -> + parse_remaining_len(Rest, #mqtt_packet_header{type = Type, + dup = bool(Dup), + qos = fixqos(Type, QoS), + retain = bool(Retain)}, Options); +parse(Bin, Cont) when is_binary(Bin), is_function(Cont) -> + Cont(Bin). + +parse_remaining_len(<<>>, Header, Options) -> + {more, fun(Bin) -> parse_remaining_len(Bin, Header, Options) end}; +parse_remaining_len(Rest, Header, Options) -> + parse_remaining_len(Rest, Header, 1, 0, Options). + +parse_remaining_len(_Bin, _Header, _Multiplier, Length, + #{max_packet_size := MaxSize}) + when Length > MaxSize -> + error(mqtt_frame_too_large); +parse_remaining_len(<<>>, Header, Multiplier, Length, Options) -> + {more, fun(Bin) -> parse_remaining_len(Bin, Header, Multiplier, Length, Options) end}; +%% Match DISCONNECT without payload +parse_remaining_len(<<0:8, Rest/binary>>, Header = #mqtt_packet_header{type = ?DISCONNECT}, 1, 0, _Options) -> + wrap(Header, #mqtt_packet_disconnect{reason_code = ?RC_SUCCESS}, Rest); +%% Match PINGREQ. +parse_remaining_len(<<0:8, Rest/binary>>, Header, 1, 0, Options) -> + parse_frame(Rest, Header, 0, Options); +%% Match PUBACK, PUBREC, PUBREL, PUBCOMP, UNSUBACK... +parse_remaining_len(<<0:1, 2:7, Rest/binary>>, Header, 1, 0, Options) -> + parse_frame(Rest, Header, 2, Options); +parse_remaining_len(<<1:1, Len:7, Rest/binary>>, Header, Multiplier, Value, Options) -> + parse_remaining_len(Rest, Header, Multiplier * ?HIGHBIT, Value + Len * Multiplier, Options); +parse_remaining_len(<<0:1, Len:7, Rest/binary>>, Header, Multiplier, Value, + Options = #{max_packet_size:= MaxSize}) -> + FrameLen = Value + Len * Multiplier, + if + FrameLen > MaxSize -> error(mqtt_frame_too_large); + true -> parse_frame(Rest, Header, FrameLen, Options) + end. + +parse_frame(Bin, Header, 0, _Options) -> + wrap(Header, Bin); + +parse_frame(Bin, Header, Length, Options) -> + case Bin of + <> -> + case parse_packet(Header, FrameBin, Options) of + {Variable, Payload} -> + wrap(Header, Variable, Payload, Rest); + Variable -> + wrap(Header, Variable, Rest) + end; + TooShortBin -> + {more, fun(BinMore) -> + parse_frame(<>, Header, Length, Options) + end} + end. + +wrap(Header, Variable, Payload, Rest) -> + {ok, #mqtt_packet{header = Header, variable = Variable, payload = Payload}, Rest}. +wrap(Header, Variable, Rest) -> + {ok, #mqtt_packet{header = Header, variable = Variable}, Rest}. +wrap(Header, Rest) -> + {ok, #mqtt_packet{header = Header}, Rest}. + +parse_packet(#mqtt_packet_header{type = ?CONNECT}, FrameBin, _Options) -> + {ProtoName, Rest} = parse_utf8_string(FrameBin), + <> = Rest, + % Note: Crash when reserved flag doesn't equal to 0, there is no strict compliance with the MQTT5.0. + <> = Rest1, + + {Properties, Rest3} = parse_properties(Rest2, ProtoVer), + {ClientId, Rest4} = parse_utf8_string(Rest3), + ConnPacket = #mqtt_packet_connect{proto_name = ProtoName, + proto_ver = ProtoVer, + is_bridge = (BridgeTag =:= 8), + clean_start = bool(CleanStart), + will_flag = bool(WillFlag), + will_qos = WillQoS, + will_retain = bool(WillRetain), + keepalive = KeepAlive, + properties = Properties, + client_id = ClientId}, + {ConnPacket1, Rest5} = parse_will_message(ConnPacket, Rest4), + {Username, Rest6} = parse_utf8_string(Rest5, bool(UsernameFlag)), + {Passsword, <<>>} = parse_utf8_string(Rest6, bool(PasswordFlag)), + ConnPacket1#mqtt_packet_connect{username = Username, password = Passsword}; + +parse_packet(#mqtt_packet_header{type = ?CONNACK}, + <>, #{version := Ver}) -> + {Properties, <<>>} = parse_properties(Rest, Ver), + #mqtt_packet_connack{ack_flags = AckFlags, + reason_code = ReasonCode, + properties = Properties}; + +parse_packet(#mqtt_packet_header{type = ?PUBLISH, qos = QoS}, Bin, + #{version := Ver}) -> + {TopicName, Rest} = parse_utf8_string(Bin), + {PacketId, Rest1} = case QoS of + ?QOS_0 -> {undefined, Rest}; + _ -> parse_packet_id(Rest) + end, + {Properties, Payload} = parse_properties(Rest1, Ver), + {#mqtt_packet_publish{topic_name = TopicName, + packet_id = PacketId, + properties = Properties}, Payload}; + +parse_packet(#mqtt_packet_header{type = PubAck}, <>, _Options) + when ?PUBACK =< PubAck, PubAck =< ?PUBCOMP -> + #mqtt_packet_puback{packet_id = PacketId, reason_code = 0}; +parse_packet(#mqtt_packet_header{type = PubAck}, <>, + #{version := Ver = ?MQTT_PROTO_V5}) + when ?PUBACK =< PubAck, PubAck =< ?PUBCOMP -> + {Properties, <<>>} = parse_properties(Rest, Ver), + #mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode, + properties = Properties}; + +parse_packet(#mqtt_packet_header{type = ?SUBSCRIBE}, <>, + #{version := Ver}) -> + {Properties, Rest1} = parse_properties(Rest, Ver), + TopicFilters = parse_topic_filters(subscribe, Rest1), + #mqtt_packet_subscribe{packet_id = PacketId, + properties = Properties, + topic_filters = TopicFilters}; + +parse_packet(#mqtt_packet_header{type = ?SUBACK}, <>, + #{version := Ver}) -> + {Properties, Rest1} = parse_properties(Rest, Ver), + #mqtt_packet_suback{packet_id = PacketId, + properties = Properties, + reason_codes = parse_reason_codes(Rest1)}; + +parse_packet(#mqtt_packet_header{type = ?UNSUBSCRIBE}, <>, + #{version := Ver}) -> + {Properties, Rest1} = parse_properties(Rest, Ver), + TopicFilters = parse_topic_filters(unsubscribe, Rest1), + #mqtt_packet_unsubscribe{packet_id = PacketId, + properties = Properties, + topic_filters = TopicFilters}; + +parse_packet(#mqtt_packet_header{type = ?UNSUBACK}, <>, _Options) -> + #mqtt_packet_unsuback{packet_id = PacketId}; +parse_packet(#mqtt_packet_header{type = ?UNSUBACK}, <>, + #{version := Ver}) -> + {Properties, Rest1} = parse_properties(Rest, Ver), + ReasonCodes = parse_reason_codes(Rest1), + #mqtt_packet_unsuback{packet_id = PacketId, + properties = Properties, + reason_codes = ReasonCodes}; + +parse_packet(#mqtt_packet_header{type = ?DISCONNECT}, <>, + #{version := ?MQTT_PROTO_V5}) -> + {Properties, <<>>} = parse_properties(Rest, ?MQTT_PROTO_V5), + #mqtt_packet_disconnect{reason_code = ReasonCode, + properties = Properties}; + +parse_packet(#mqtt_packet_header{type = ?AUTH}, <>, + #{version := ?MQTT_PROTO_V5}) -> + {Properties, <<>>} = parse_properties(Rest, ?MQTT_PROTO_V5), + #mqtt_packet_auth{reason_code = ReasonCode, properties = Properties}. + +parse_will_message(Packet = #mqtt_packet_connect{will_flag = true, + proto_ver = Ver}, Bin) -> + {Props, Rest} = parse_properties(Bin, Ver), + {Topic, Rest1} = parse_utf8_string(Rest), + {Payload, Rest2} = parse_binary_data(Rest1), + {Packet#mqtt_packet_connect{will_props = Props, + will_topic = Topic, + will_payload = Payload}, Rest2}; +parse_will_message(Packet, Bin) -> + {Packet, Bin}. + +% protocol_approved(Ver, Name) -> +% lists:member({Ver, Name}, ?PROTOCOL_NAMES). + +parse_packet_id(<>) -> + {PacketId, Rest}. + +parse_properties(Bin, Ver) when Ver =/= ?MQTT_PROTO_V5 -> + {undefined, Bin}; +%% TODO: version mess? +parse_properties(<<>>, ?MQTT_PROTO_V5) -> + {#{}, <<>>}; +parse_properties(<<0, Rest/binary>>, ?MQTT_PROTO_V5) -> + {#{}, Rest}; +parse_properties(Bin, ?MQTT_PROTO_V5) -> + {Len, Rest} = parse_variable_byte_integer(Bin), + <> = Rest, + {parse_property(PropsBin, #{}), Rest1}. + +parse_property(<<>>, Props) -> + Props; +parse_property(<<16#01, Val, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Payload-Format-Indicator' => Val}); +parse_property(<<16#02, Val:32/big, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Message-Expiry-Interval' => Val}); +parse_property(<<16#03, Bin/binary>>, Props) -> + {Val, Rest} = parse_utf8_string(Bin), + parse_property(Rest, Props#{'Content-Type' => Val}); +parse_property(<<16#08, Bin/binary>>, Props) -> + {Val, Rest} = parse_utf8_string(Bin), + parse_property(Rest, Props#{'Response-Topic' => Val}); +parse_property(<<16#09, Len:16/big, Val:Len/binary, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Correlation-Data' => Val}); +parse_property(<<16#0B, Bin/binary>>, Props) -> + {Val, Rest} = parse_variable_byte_integer(Bin), + parse_property(Rest, Props#{'Subscription-Identifier' => Val}); +parse_property(<<16#11, Val:32/big, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Session-Expiry-Interval' => Val}); +parse_property(<<16#12, Bin/binary>>, Props) -> + {Val, Rest} = parse_utf8_string(Bin), + parse_property(Rest, Props#{'Assigned-Client-Identifier' => Val}); +parse_property(<<16#13, Val:16, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Server-Keep-Alive' => Val}); +parse_property(<<16#15, Bin/binary>>, Props) -> + {Val, Rest} = parse_utf8_string(Bin), + parse_property(Rest, Props#{'Authentication-Method' => Val}); +parse_property(<<16#16, Len:16/big, Val:Len/binary, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Authentication-Data' => Val}); +parse_property(<<16#17, Val, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Request-Problem-Information' => Val}); +parse_property(<<16#18, Val:32, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Will-Delay-Interval' => Val}); +parse_property(<<16#19, Val, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Request-Response-Information' => Val}); +parse_property(<<16#1A, Bin/binary>>, Props) -> + {Val, Rest} = parse_utf8_string(Bin), + parse_property(Rest, Props#{'Response-Information' => Val}); +parse_property(<<16#1C, Bin/binary>>, Props) -> + {Val, Rest} = parse_utf8_string(Bin), + parse_property(Rest, Props#{'Server-Reference' => Val}); +parse_property(<<16#1F, Bin/binary>>, Props) -> + {Val, Rest} = parse_utf8_string(Bin), + parse_property(Rest, Props#{'Reason-String' => Val}); +parse_property(<<16#21, Val:16/big, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Receive-Maximum' => Val}); +parse_property(<<16#22, Val:16/big, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Topic-Alias-Maximum' => Val}); +parse_property(<<16#23, Val:16/big, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Topic-Alias' => Val}); +parse_property(<<16#24, Val, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Maximum-QoS' => Val}); +parse_property(<<16#25, Val, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Retain-Available' => Val}); +parse_property(<<16#26, Bin/binary>>, Props) -> + {Pair, Rest} = parse_utf8_pair(Bin), + case maps:find('User-Property', Props) of + {ok, UserProps} -> + parse_property(Rest,Props#{'User-Property' := [Pair|UserProps]}); + error -> + parse_property(Rest, Props#{'User-Property' => [Pair]}) + end; +parse_property(<<16#27, Val:32, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Maximum-Packet-Size' => Val}); +parse_property(<<16#28, Val, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Wildcard-Subscription-Available' => Val}); +parse_property(<<16#29, Val, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Subscription-Identifier-Available' => Val}); +parse_property(<<16#2A, Val, Bin/binary>>, Props) -> + parse_property(Bin, Props#{'Shared-Subscription-Available' => Val}). + +parse_variable_byte_integer(Bin) -> + parse_variable_byte_integer(Bin, 1, 0). +parse_variable_byte_integer(<<1:1, Len:7, Rest/binary>>, Multiplier, Value) -> + parse_variable_byte_integer(Rest, Multiplier * ?HIGHBIT, Value + Len * Multiplier); +parse_variable_byte_integer(<<0:1, Len:7, Rest/binary>>, Multiplier, Value) -> + {Value + Len * Multiplier, Rest}. + +parse_topic_filters(subscribe, Bin) -> + [{Topic, #{rh => Rh, rap => Rap, nl => Nl, qos => QoS, rc => 0}} + || <> <= Bin]; + +parse_topic_filters(unsubscribe, Bin) -> + [Topic || <> <= Bin]. + +parse_reason_codes(Bin) -> + [Code || <> <= Bin]. + +parse_utf8_pair(<>) -> + {{Key, Val}, Rest}. + +parse_utf8_string(Bin, false) -> + {undefined, Bin}; +parse_utf8_string(Bin, true) -> + parse_utf8_string(Bin). + +parse_utf8_string(<>) -> + {Str, Rest}. + +parse_binary_data(<>) -> + {Data, Rest}. + +%%------------------------------------------------------------------------------ +%% Serialize MQTT Packet +%%------------------------------------------------------------------------------ + +-spec(serialize(emqx_mqtt_types:packet()) -> iodata()). +serialize(Packet) -> + serialize(Packet, ?DEFAULT_OPTIONS). + +-spec(serialize(emqx_mqtt_types:packet(), options()) -> iodata()). +serialize(#mqtt_packet{header = Header, + variable = Variable, + payload = Payload}, Options) when is_map(Options) -> + serialize(Header, serialize_variable(Variable, merge_opts(Options)), serialize_payload(Payload)). + +serialize(#mqtt_packet_header{type = Type, + dup = Dup, + qos = QoS, + retain = Retain}, VariableBin, PayloadBin) + when ?CONNECT =< Type andalso Type =< ?AUTH -> + Len = iolist_size(VariableBin) + iolist_size(PayloadBin), + (Len =< ?MAX_PACKET_SIZE) orelse error(mqtt_frame_too_large), + [<>, + serialize_remaining_len(Len), VariableBin, PayloadBin]. + +serialize_variable(#mqtt_packet_connect{ + proto_name = ProtoName, + proto_ver = ProtoVer, + is_bridge = IsBridge, + clean_start = CleanStart, + will_flag = WillFlag, + will_qos = WillQoS, + will_retain = WillRetain, + keepalive = KeepAlive, + properties = Properties, + client_id = ClientId, + will_props = WillProps, + will_topic = WillTopic, + will_payload = WillPayload, + username = Username, + password = Password}, _Options) -> + [serialize_binary_data(ProtoName), + <<(case IsBridge of + true -> 16#80 + ProtoVer; + false -> ProtoVer + end):8, + (flag(Username)):1, + (flag(Password)):1, + (flag(WillRetain)):1, + WillQoS:2, + (flag(WillFlag)):1, + (flag(CleanStart)):1, + 0:1, + KeepAlive:16/big-unsigned-integer>>, + serialize_properties(Properties, ProtoVer), + serialize_utf8_string(ClientId), + case WillFlag of + true -> [serialize_properties(WillProps, ProtoVer), + serialize_utf8_string(WillTopic), + serialize_binary_data(WillPayload)]; + false -> <<>> + end, + serialize_utf8_string(Username, true), + serialize_utf8_string(Password, true)]; + +serialize_variable(#mqtt_packet_connack{ack_flags = AckFlags, + reason_code = ReasonCode, + properties = Properties}, + #{version := Ver}) -> + [AckFlags, ReasonCode, serialize_properties(Properties, Ver)]; + +serialize_variable(#mqtt_packet_publish{topic_name = TopicName, + packet_id = PacketId, + properties = Properties}, + #{version := Ver}) -> + [serialize_utf8_string(TopicName), + if + PacketId =:= undefined -> <<>>; + true -> <> + end, + serialize_properties(Properties, Ver)]; + +serialize_variable(#mqtt_packet_puback{packet_id = PacketId}, + #{version := Ver}) + when Ver == ?MQTT_PROTO_V3; Ver == ?MQTT_PROTO_V4 -> + <>; +serialize_variable(#mqtt_packet_puback{packet_id = PacketId, + reason_code = ReasonCode, + properties = Properties}, + #{version := ?MQTT_PROTO_V5}) -> + [<>, ReasonCode, + serialize_properties(Properties, ?MQTT_PROTO_V5)]; + +serialize_variable(#mqtt_packet_subscribe{packet_id = PacketId, + properties = Properties, + topic_filters = TopicFilters}, + #{version := Ver}) -> + [<>, serialize_properties(Properties, Ver), + serialize_topic_filters(subscribe, TopicFilters, Ver)]; + +serialize_variable(#mqtt_packet_suback{packet_id = PacketId, + properties = Properties, + reason_codes = ReasonCodes}, + #{version := Ver}) -> + [<>, serialize_properties(Properties, Ver), + serialize_reason_codes(ReasonCodes)]; + +serialize_variable(#mqtt_packet_unsubscribe{packet_id = PacketId, + properties = Properties, + topic_filters = TopicFilters}, + #{version := Ver}) -> + [<>, serialize_properties(Properties, Ver), + serialize_topic_filters(unsubscribe, TopicFilters, Ver)]; + +serialize_variable(#mqtt_packet_unsuback{packet_id = PacketId, + properties = Properties, + reason_codes = ReasonCodes}, + #{version := Ver}) -> + [<>, serialize_properties(Properties, Ver), + serialize_reason_codes(ReasonCodes)]; + +serialize_variable(#mqtt_packet_disconnect{}, #{version := Ver}) + when Ver == ?MQTT_PROTO_V3; Ver == ?MQTT_PROTO_V4 -> + <<>>; + +serialize_variable(#mqtt_packet_disconnect{reason_code = ReasonCode, + properties = Properties}, + #{version := Ver = ?MQTT_PROTO_V5}) -> + [ReasonCode, serialize_properties(Properties, Ver)]; +serialize_variable(#mqtt_packet_disconnect{}, _Ver) -> + <<>>; + +serialize_variable(#mqtt_packet_auth{reason_code = ReasonCode, + properties = Properties}, + #{version := Ver = ?MQTT_PROTO_V5}) -> + [ReasonCode, serialize_properties(Properties, Ver)]; + +serialize_variable(PacketId, ?MQTT_PROTO_V3) when is_integer(PacketId) -> + <>; +serialize_variable(PacketId, ?MQTT_PROTO_V4) when is_integer(PacketId) -> + <>; +serialize_variable(undefined, _Ver) -> + <<>>. + +serialize_payload(undefined) -> <<>>; +serialize_payload(Bin) -> Bin. + +serialize_properties(_Props, Ver) when Ver =/= ?MQTT_PROTO_V5 -> + <<>>; +serialize_properties(Props, ?MQTT_PROTO_V5) -> + serialize_properties(Props). + +serialize_properties(undefined) -> + <<0>>; +serialize_properties(Props) when map_size(Props) == 0 -> + <<0>>; +serialize_properties(Props) when is_map(Props) -> + Bin = << <<(serialize_property(Prop, Val))/binary>> || {Prop, Val} <- maps:to_list(Props) >>, + [serialize_variable_byte_integer(byte_size(Bin)), Bin]. + +serialize_property(_, undefined) -> + <<>>; +serialize_property('Payload-Format-Indicator', Val) -> + <<16#01, Val>>; +serialize_property('Message-Expiry-Interval', Val) -> + <<16#02, Val:32/big>>; +serialize_property('Content-Type', Val) -> + <<16#03, (serialize_utf8_string(Val))/binary>>; +serialize_property('Response-Topic', Val) -> + <<16#08, (serialize_utf8_string(Val))/binary>>; +serialize_property('Correlation-Data', Val) -> + <<16#09, (byte_size(Val)):16, Val/binary>>; +serialize_property('Subscription-Identifier', Val) -> + <<16#0B, (serialize_variable_byte_integer(Val))/binary>>; +serialize_property('Session-Expiry-Interval', Val) -> + <<16#11, Val:32/big>>; +serialize_property('Assigned-Client-Identifier', Val) -> + <<16#12, (serialize_utf8_string(Val))/binary>>; +serialize_property('Server-Keep-Alive', Val) -> + <<16#13, Val:16/big>>; +serialize_property('Authentication-Method', Val) -> + <<16#15, (serialize_utf8_string(Val))/binary>>; +serialize_property('Authentication-Data', Val) -> + <<16#16, (iolist_size(Val)):16, Val/binary>>; +serialize_property('Request-Problem-Information', Val) -> + <<16#17, Val>>; +serialize_property('Will-Delay-Interval', Val) -> + <<16#18, Val:32/big>>; +serialize_property('Request-Response-Information', Val) -> + <<16#19, Val>>; +serialize_property('Response-Information', Val) -> + <<16#1A, (serialize_utf8_string(Val))/binary>>; +serialize_property('Server-Reference', Val) -> + <<16#1C, (serialize_utf8_string(Val))/binary>>; +serialize_property('Reason-String', Val) -> + <<16#1F, (serialize_utf8_string(Val))/binary>>; +serialize_property('Receive-Maximum', Val) -> + <<16#21, Val:16/big>>; +serialize_property('Topic-Alias-Maximum', Val) -> + <<16#22, Val:16/big>>; +serialize_property('Topic-Alias', Val) -> + <<16#23, Val:16/big>>; +serialize_property('Maximum-QoS', Val) -> + <<16#24, Val>>; +serialize_property('Retain-Available', Val) -> + <<16#25, Val>>; +serialize_property('User-Property', {Key, Val}) -> + <<16#26, (serialize_utf8_pair({Key, Val}))/binary>>; +serialize_property('User-Property', Props) when is_list(Props) -> + << <<(serialize_property('User-Property', {Key, Val}))/binary>> + || {Key, Val} <- Props >>; +serialize_property('Maximum-Packet-Size', Val) -> + <<16#27, Val:32/big>>; +serialize_property('Wildcard-Subscription-Available', Val) -> + <<16#28, Val>>; +serialize_property('Subscription-Identifier-Available', Val) -> + <<16#29, Val>>; +serialize_property('Shared-Subscription-Available', Val) -> + <<16#2A, Val>>. + +serialize_topic_filters(subscribe, TopicFilters, ?MQTT_PROTO_V5) -> + << <<(serialize_utf8_string(Topic))/binary, + ?RESERVED:2, Rh:2, (flag(Rap)):1,(flag(Nl)):1, QoS:2 >> + || {Topic, #{rh := Rh, rap := Rap, nl := Nl, qos := QoS}} + <- TopicFilters >>; + +serialize_topic_filters(subscribe, TopicFilters, _Ver) -> + << <<(serialize_utf8_string(Topic))/binary, ?RESERVED:6, QoS:2>> + || {Topic, #{qos := QoS}} <- TopicFilters >>; + +serialize_topic_filters(unsubscribe, TopicFilters, _Ver) -> + << <<(serialize_utf8_string(Topic))/binary>> || Topic <- TopicFilters >>. + +serialize_reason_codes(undefined) -> + <<>>; +serialize_reason_codes(ReasonCodes) when is_list(ReasonCodes) -> + << <> || Code <- ReasonCodes >>. + +serialize_utf8_pair({Name, Value}) -> + << (serialize_utf8_string(Name))/binary, (serialize_utf8_string(Value))/binary >>. + +serialize_binary_data(Bin) -> + [<<(byte_size(Bin)):16/big-unsigned-integer>>, Bin]. + +serialize_utf8_string(undefined, false) -> + error(utf8_string_undefined); +serialize_utf8_string(undefined, true) -> + <<>>; +serialize_utf8_string(String, _AllowNull) -> + serialize_utf8_string(String). + +serialize_utf8_string(String) -> + StringBin = unicode:characters_to_binary(String), + Len = byte_size(StringBin), + true = (Len =< 16#ffff), + <>. + +serialize_remaining_len(I) -> + serialize_variable_byte_integer(I). + +serialize_variable_byte_integer(N) when N =< ?LOWBITS -> + <<0:1, N:7>>; +serialize_variable_byte_integer(N) -> + <<1:1, (N rem ?HIGHBIT):7, (serialize_variable_byte_integer(N div ?HIGHBIT))/binary>>. + +bool(0) -> false; +bool(1) -> true. + +flag(undefined) -> ?RESERVED; +flag(false) -> 0; +flag(true) -> 1; +flag(X) when is_integer(X) -> X; +flag(B) when is_binary(B) -> 1. + +fixqos(?PUBREL, 0) -> 1; +fixqos(?SUBSCRIBE, 0) -> 1; +fixqos(?UNSUBSCRIBE, 0) -> 1; +fixqos(_Type, QoS) -> QoS. diff --git a/src/emqx_gc.erl b/src/emqx_gc.erl new file mode 100644 index 000000000..d608954a0 --- /dev/null +++ b/src/emqx_gc.erl @@ -0,0 +1,105 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc This module manages an opaque collection of statistics data used to +%% force garbage collection on `self()' process when hitting thresholds. +%% Namely: +%% (1) Total number of messages passed through +%% (2) Total data volume passed through +%% @end + +-module(emqx_gc). + +-export([init/1, run/3, info/1, reset/1]). + +-type(opts() :: #{count => integer(), + bytes => integer()}). + +-type(st() :: #{cnt => {integer(), integer()}, + oct => {integer(), integer()}}). + +-type(gc_state() :: {?MODULE, st()}). + +-define(disabled, disabled). +-define(ENABLED(X), (is_integer(X) andalso X > 0)). + +%% @doc Initialize force GC state. +-spec(init(opts() | false) -> gc_state() | undefined). +init(#{count := Count, bytes := Bytes}) -> + Cnt = [{cnt, {Count, Count}} || ?ENABLED(Count)], + Oct = [{oct, {Bytes, Bytes}} || ?ENABLED(Bytes)], + {?MODULE, maps:from_list(Cnt ++ Oct)}; +init(false) -> undefined. + +%% @doc Try to run GC based on reduntions of count or bytes. +-spec(run(pos_integer(), pos_integer(), gc_state()) -> {boolean(), gc_state()}). +run(Cnt, Oct, {?MODULE, St}) -> + {Res, St1} = run([{cnt, Cnt}, {oct, Oct}], St), + {Res, {?MODULE, St1}}; +run(_Cnt, _Oct, undefined) -> + {false, undefined}. + +run([], St) -> + {false, St}; +run([{K, N}|T], St) -> + case dec(K, N, St) of + {true, St1} -> + {true, do_gc(St1)}; + {false, St1} -> + run(T, St1) + end. + +%% @doc Info of GC state. +-spec(info(gc_state()) -> map() | undefined). +info({?MODULE, St}) -> + St; +info(undefined) -> + undefined. + +%% @doc Reset counters to zero. +-spec(reset(gc_state()) -> gc_state()). +reset({?MODULE, St}) -> + {?MODULE, do_reset(St)}; +reset(undefined) -> + undefined. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +-spec(dec(cnt | oct, pos_integer(), st()) -> {boolean(), st()}). +dec(Key, Num, St) -> + case maps:get(Key, St, ?disabled) of + ?disabled -> + {false, St}; + {Init, Remain} when Remain > Num -> + {false, maps:put(Key, {Init, Remain - Num}, St)}; + _ -> + {true, St} + end. + +do_gc(St) -> + true = erlang:garbage_collect(), + do_reset(St). + +do_reset(St) -> + do_reset(cnt, do_reset(oct, St)). + +%% Reset counters to zero. +do_reset(Key, St) -> + case maps:get(Key, St, ?disabled) of + ?disabled -> St; + {Init, _} -> maps:put(Key, {Init, Init}, St) + end. + diff --git a/src/emqttd_gen_mod.erl b/src/emqx_gen_mod.erl similarity index 71% rename from src/emqttd_gen_mod.erl rename to src/emqx_gen_mod.erl index 012b610da..42466fdd9 100644 --- a/src/emqttd_gen_mod.erl +++ b/src/emqx_gen_mod.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,15 +11,8 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- -%% @doc emqttd gen_mod behaviour - --module(emqttd_gen_mod). - --author("Feng Lee "). - --include("emqttd.hrl"). +-module(emqx_gen_mod). -ifdef(use_specs). diff --git a/src/emqttd_guid.erl b/src/emqx_guid.erl similarity index 91% rename from src/emqttd_guid.erl rename to src/emqx_guid.erl index 805a128b6..fa9139ebd 100644 --- a/src/emqttd_guid.erl +++ b/src/emqx_guid.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,15 +11,14 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- %% @doc Generate global unique id for mqtt message. %% %% -------------------------------------------------------- -%% | Timestamp | NodeID + PID | Sequence | +%% | Timestamp | NodeID + PID | Sequence | %% |<------- 64bits ------->|<--- 48bits --->|<- 16bits ->| %% -------------------------------------------------------- -%% +%% %% 1. Timestamp: erlang:system_time if Erlang >= R18, otherwise os:timestamp %% 2. NodeId: encode node() to 2 bytes integer %% 3. Pid: encode pid to 4 bytes integer @@ -28,7 +26,7 @@ %% %% @end --module(emqttd_guid). +-module(emqx_guid). -export([gen/0, new/0, timestamp/1, to_hexstr/1, from_hexstr/1, to_base62/1, from_base62/1]). @@ -128,8 +126,9 @@ from_hexstr(S) -> I = list_to_integer(binary_to_list(S), 16), <>. to_base62(<>) -> - emqttd_base62:encode(I). + emqx_base62:encode(I). from_base62(S) -> - I = emqttd_base62:decode(S), <>. + I = emqx_base62:decode(S, integer), + <>. diff --git a/src/emqx_hooks.erl b/src/emqx_hooks.erl new file mode 100644 index 000000000..55b2476ee --- /dev/null +++ b/src/emqx_hooks.erl @@ -0,0 +1,212 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_hooks). + +-behaviour(gen_server). + +-export([start_link/0, stop/0]). + +%% Hooks API +-export([add/2, add/3, add/4, del/2, run/2, run/3, lookup/1]). + +%% gen_server Function Exports +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-type(hookpoint() :: atom()). +-type(action() :: function() | mfa()). +-type(filter() :: function() | mfa()). + +-record(callback, {action :: action(), + filter :: filter(), + priority :: integer()}). + +-record(hook, {name :: hookpoint(), callbacks :: list(#callback{})}). + +-export_type([hookpoint/0, action/0, filter/0]). + +-define(TAB, ?MODULE). +-define(SERVER, ?MODULE). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], [{hibernate_after, 1000}]). + +-spec(stop() -> ok). +stop() -> + gen_server:stop(?SERVER, normal, infinity). + +%%------------------------------------------------------------------------------ +%% Hooks API +%%------------------------------------------------------------------------------ + +%% @doc Register a callback +-spec(add(hookpoint(), action() | #callback{}) -> emqx_types:ok_or_error(already_exists)). +add(HookPoint, Callback) when is_record(Callback, callback) -> + gen_server:call(?SERVER, {add, HookPoint, Callback}, infinity); +add(HookPoint, Action) when is_function(Action); is_tuple(Action) -> + add(HookPoint, #callback{action = Action, priority = 0}). + +-spec(add(hookpoint(), action(), filter() | integer() | list()) + -> emqx_types:ok_or_error(already_exists)). +add(HookPoint, Action, InitArgs) when is_function(Action), is_list(InitArgs) -> + add(HookPoint, #callback{action = {Action, InitArgs}, priority = 0}); +add(HookPoint, Action, Filter) when is_function(Filter); is_tuple(Filter) -> + add(HookPoint, #callback{action = Action, filter = Filter, priority = 0}); +add(HookPoint, Action, Priority) when is_integer(Priority) -> + add(HookPoint, #callback{action = Action, priority = Priority}). + +-spec(add(hookpoint(), action(), filter(), integer()) + -> emqx_types:ok_or_error(already_exists)). +add(HookPoint, Action, Filter, Priority) -> + add(HookPoint, #callback{action = Action, filter = Filter, priority = Priority}). + +%% @doc Unregister a callback. +-spec(del(hookpoint(), action()) -> ok). +del(HookPoint, Action) -> + gen_server:cast(?SERVER, {del, HookPoint, Action}). + +%% @doc Run hooks. +-spec(run(atom(), list(Arg :: any())) -> ok | stop). +run(HookPoint, Args) -> + run_(lookup(HookPoint), Args). + +%% @doc Run hooks with Accumulator. +-spec(run(atom(), list(Arg :: any()), any()) -> any()). +run(HookPoint, Args, Acc) -> + run_(lookup(HookPoint), Args, Acc). + +%% @private +run_([#callback{action = Action, filter = Filter} | Callbacks], Args) -> + case filtered(Filter, Args) orelse execute(Action, Args) of + true -> run_(Callbacks, Args); + ok -> run_(Callbacks, Args); + stop -> stop; + _Any -> run_(Callbacks, Args) + end; +run_([], _Args) -> + ok. + +%% @private +run_([#callback{action = Action, filter = Filter} | Callbacks], Args, Acc) -> + Args1 = Args ++ [Acc], + case filtered(Filter, Args1) orelse execute(Action, Args1) of + true -> run_(Callbacks, Args, Acc); + ok -> run_(Callbacks, Args, Acc); + {ok, NewAcc} -> run_(Callbacks, Args, NewAcc); + stop -> {stop, Acc}; + {stop, NewAcc} -> {stop, NewAcc}; + _Any -> run_(Callbacks, Args, Acc) + end; +run_([], _Args, Acc) -> + {ok, Acc}. + +filtered(undefined, _Args) -> + false; +filtered(Filter, Args) -> + execute(Filter, Args). + +execute(Action, Args) when is_function(Action) -> + erlang:apply(Action, Args); +execute({Fun, InitArgs}, Args) when is_function(Fun) -> + erlang:apply(Fun, Args ++ InitArgs); +execute({M, F, A}, Args) -> + erlang:apply(M, F, Args ++ A). + +%% @doc Lookup callbacks. +-spec(lookup(hookpoint()) -> [#callback{}]). +lookup(HookPoint) -> + case ets:lookup(?TAB, HookPoint) of + [#hook{callbacks = Callbacks}] -> + Callbacks; + [] -> [] + end. + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + ok = emqx_tables:new(?TAB, [{keypos, #hook.name}, {read_concurrency, true}]), + {ok, #{}}. + +handle_call({add, HookPoint, Callback = #callback{action = Action}}, _From, State) -> + Reply = case lists:keymember(Action, 2, Callbacks = lookup(HookPoint)) of + true -> + {error, already_exists}; + false -> + insert_hook(HookPoint, add_callback(Callback, Callbacks)) + end, + {reply, Reply, State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[Hooks] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({del, HookPoint, Action}, State) -> + case del_callback(Action, lookup(HookPoint)) of + [] -> + ets:delete(?TAB, HookPoint); + Callbacks -> + insert_hook(HookPoint, Callbacks) + end, + {noreply, State}; + +handle_cast(Msg, State) -> + emqx_logger:error("[Hooks] unexpected msg: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + emqx_logger:error("[Hooks] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +insert_hook(HookPoint, Callbacks) -> + ets:insert(?TAB, #hook{name = HookPoint, callbacks = Callbacks}), ok. + +add_callback(C, Callbacks) -> + add_callback(C, Callbacks, []). + +add_callback(C, [], Acc) -> + lists:reverse([C|Acc]); +add_callback(C1 = #callback{priority = P1}, [C2 = #callback{priority = P2}|More], Acc) + when P1 =< P2 -> + add_callback(C1, More, [C2|Acc]); +add_callback(C1, More, Acc) -> + lists:append(lists:reverse(Acc), [C1 | More]). + +del_callback(Action, Callbacks) -> + del_callback(Action, Callbacks, []). + +del_callback(_Action, [], Acc) -> + lists:reverse(Acc); +del_callback(Action, [#callback{action = Action} | Callbacks], Acc) -> + del_callback(Action, Callbacks, Acc); +del_callback(Action = {M, F}, [#callback{action = {M, F, _A}} | Callbacks], Acc) -> + del_callback(Action, Callbacks, Acc); +del_callback(Func, [#callback{action = {Func, _A}} | Callbacks], Acc) -> + del_callback(Func, Callbacks, Acc); +del_callback(Action, [Callback | Callbacks], Acc) -> + del_callback(Action, Callbacks, [Callback | Acc]). + diff --git a/src/emqx_inflight.erl b/src/emqx_inflight.erl new file mode 100644 index 000000000..876052974 --- /dev/null +++ b/src/emqx_inflight.erl @@ -0,0 +1,93 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_inflight). + +-export([new/1, contain/2, lookup/2, insert/3, update/3, update_size/2, delete/2, values/1, + to_list/1, size/1, max_size/1, is_full/1, is_empty/1, window/1]). + +-type(max_size() :: pos_integer()). +-type(inflight() :: {?MODULE, max_size(), gb_trees:tree()}). + +-export_type([inflight/0]). + +-spec(new(non_neg_integer()) -> inflight()). +new(MaxSize) when MaxSize >= 0 -> + {?MODULE, MaxSize, gb_trees:empty()}. + +-spec(contain(Key :: term(), inflight()) -> boolean()). +contain(Key, {?MODULE, _MaxSize, Tree}) -> + gb_trees:is_defined(Key, Tree). + +-spec(lookup(Key :: term(), inflight()) -> {value, term()} | none). +lookup(Key, {?MODULE, _MaxSize, Tree}) -> + gb_trees:lookup(Key, Tree). + +-spec(insert(Key :: term(), Value :: term(), inflight()) -> inflight()). +insert(Key, Value, {?MODULE, MaxSize, Tree}) -> + {?MODULE, MaxSize, gb_trees:insert(Key, Value, Tree)}. + +-spec(delete(Key :: term(), inflight()) -> inflight()). +delete(Key, {?MODULE, MaxSize, Tree}) -> + {?MODULE, MaxSize, gb_trees:delete(Key, Tree)}. + +-spec(update(Key :: term(), Val :: term(), inflight()) -> inflight()). +update(Key, Val, {?MODULE, MaxSize, Tree}) -> + {?MODULE, MaxSize, gb_trees:update(Key, Val, Tree)}. + +-spec(update_size(integer(), inflight()) -> inflight()). +update_size(MaxSize, {?MODULE, _OldMaxSize, Tree}) -> + {?MODULE, MaxSize, Tree}. + +-spec(is_full(inflight()) -> boolean()). +is_full({?MODULE, 0, _Tree}) -> + false; +is_full({?MODULE, MaxSize, Tree}) -> + MaxSize =< gb_trees:size(Tree). + +-spec(is_empty(inflight()) -> boolean()). +is_empty({?MODULE, _MaxSize, Tree}) -> + gb_trees:is_empty(Tree). + +-spec(smallest(inflight()) -> {K :: term(), V :: term()}). +smallest({?MODULE, _MaxSize, Tree}) -> + gb_trees:smallest(Tree). + +-spec(largest(inflight()) -> {K :: term(), V :: term()}). +largest({?MODULE, _MaxSize, Tree}) -> + gb_trees:largest(Tree). + +-spec(values(inflight()) -> list()). +values({?MODULE, _MaxSize, Tree}) -> + gb_trees:values(Tree). + +-spec(to_list(inflight()) -> list({K :: term(), V :: term()})). +to_list({?MODULE, _MaxSize, Tree}) -> + gb_trees:to_list(Tree). + +-spec(window(inflight()) -> list()). +window(Inflight = {?MODULE, _MaxSize, Tree}) -> + case gb_trees:is_empty(Tree) of + true -> []; + false -> [Key || {Key, _Val} <- [smallest(Inflight), largest(Inflight)]] + end. + +-spec(size(inflight()) -> non_neg_integer()). +size({?MODULE, _MaxSize, Tree}) -> + gb_trees:size(Tree). + +-spec(max_size(inflight()) -> non_neg_integer()). +max_size({?MODULE, MaxSize, _Tree}) -> + MaxSize. + diff --git a/src/emqx_json.erl b/src/emqx_json.erl new file mode 100644 index 000000000..9c0740b33 --- /dev/null +++ b/src/emqx_json.erl @@ -0,0 +1,65 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_json). + +-export([encode/1, encode/2, safe_encode/1, safe_encode/2]). +-export([decode/1, decode/2, safe_decode/1, safe_decode/2]). + +-spec(encode(jsx:json_term()) -> jsx:json_text()). +encode(Term) -> + jsx:encode(Term). + +-spec(encode(jsx:json_term(), jsx_to_json:config()) -> jsx:json_text()). +encode(Term, Opts) -> + jsx:encode(Term, Opts). + +-spec(safe_encode(jsx:json_term()) + -> {ok, jsx:json_text()} | {error, term()}). +safe_encode(Term) -> + safe_encode(Term, []). + +-spec(safe_encode(jsx:json_term(), jsx_to_json:config()) + -> {ok, jsx:json_text()} | {error, term()}). +safe_encode(Term, Opts) -> + try encode(Term, Opts) of + Json -> {ok, Json} + catch + error:Reason -> + {error, Reason} + end. + +-spec(decode(jsx:json_text()) -> jsx:json_term()). +decode(Json) -> + jsx:decode(Json). + +-spec(decode(jsx:json_text(), jsx_to_json:config()) -> jsx:json_term()). +decode(Json, Opts) -> + jsx:decode(Json, Opts). + +-spec(safe_decode(jsx:json_text()) + -> {ok, jsx:json_term()} | {error, term()}). +safe_decode(Json) -> + safe_decode(Json, []). + +-spec(safe_decode(jsx:json_text(), jsx_to_json:config()) + -> {ok, jsx:json_term()} | {error, term()}). +safe_decode(Json, Opts) -> + try decode(Json, Opts) of + Term -> {ok, Term} + catch + error:Reason -> + {error, Reason} + end. + diff --git a/src/emqttd_keepalive.erl b/src/emqx_keepalive.erl similarity index 79% rename from src/emqttd_keepalive.erl rename to src/emqx_keepalive.erl index abc6dbc50..25740b099 100644 --- a/src/emqttd_keepalive.erl +++ b/src/emqx_keepalive.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,13 +11,8 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- -%% @doc Client Keepalive - --module(emqttd_keepalive). - --author("Feng Lee "). +-module(emqx_keepalive). -export([start/3, check/1, cancel/1]). @@ -33,19 +27,21 @@ start(_, 0, _) -> {ok, #keepalive{}}; start(StatFun, TimeoutSec, TimeoutMsg) -> - case StatFun() of + case catch StatFun() of {ok, StatVal} -> {ok, #keepalive{statfun = StatFun, statval = StatVal, tsec = TimeoutSec, tmsg = TimeoutMsg, tref = timer(TimeoutSec, TimeoutMsg)}}; {error, Error} -> - {error, Error} + {error, Error}; + {'EXIT', Reason} -> + {error, Reason} end. -%% @doc Check keepalive, called when timeout. +%% @doc Check keepalive, called when timeout... -spec(check(keepalive()) -> {ok, keepalive()} | {error, term()}). check(KeepAlive = #keepalive{statfun = StatFun, statval = LastVal, repeat = Repeat}) -> - case StatFun() of + case catch StatFun() of {ok, NewVal} -> if NewVal =/= LastVal -> {ok, resume(KeepAlive#keepalive{statval = NewVal, repeat = 0})}; @@ -55,9 +51,12 @@ check(KeepAlive = #keepalive{statfun = StatFun, statval = LastVal, repeat = Repe {error, timeout} end; {error, Error} -> - {error, Error} + {error, Error}; + {'EXIT', Reason} -> + {error, Reason} end. +-spec(resume(keepalive()) -> keepalive()). resume(KeepAlive = #keepalive{tsec = TimeoutSec, tmsg = TimeoutMsg}) -> KeepAlive#keepalive{tref = timer(TimeoutSec, TimeoutMsg)}. @@ -68,6 +67,6 @@ cancel(#keepalive{tref = TRef}) when is_reference(TRef) -> cancel(_) -> ok. -timer(Sec, Msg) -> - erlang:send_after(timer:seconds(Sec), self(), Msg). +timer(Secs, Msg) -> + erlang:send_after(timer:seconds(Secs), self(), Msg). diff --git a/src/emqx_kernel_sup.erl b/src/emqx_kernel_sup.erl new file mode 100644 index 000000000..40ec7cfd7 --- /dev/null +++ b/src/emqx_kernel_sup.erl @@ -0,0 +1,52 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_kernel_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + {ok, {{one_for_one, 10, 100}, + [child_spec(emqx_pool, supervisor), + child_spec(emqx_alarm_mgr, worker), + child_spec(emqx_hooks, worker), + child_spec(emqx_stats, worker), + child_spec(emqx_metrics, worker), + child_spec(emqx_ctl, worker), + child_spec(emqx_zone, worker), + child_spec(emqx_tracer, worker)]}}. + +child_spec(M, worker) -> + #{id => M, + start => {M, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [M]}; +child_spec(M, supervisor) -> + #{id => M, + start => {M, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [M]}. + + diff --git a/src/emqx_listeners.erl b/src/emqx_listeners.erl new file mode 100644 index 000000000..6987f03f9 --- /dev/null +++ b/src/emqx_listeners.erl @@ -0,0 +1,153 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc Start/Stop MQTT listeners. +-module(emqx_listeners). + +-include("emqx_mqtt.hrl"). + +-export([start/0, restart/0, stop/0]). +-export([start_listener/1, start_listener/3]). +-export([restart_listener/1, restart_listener/3]). +-export([stop_listener/1, stop_listener/3]). + +-type(listener() :: {esockd:proto(), esockd:listen_on(), [esockd:option()]}). + +%% @doc Start all listeners. +-spec(start() -> ok). +start() -> + lists:foreach(fun start_listener/1, emqx_config:get_env(listeners, [])). + +-spec(start_listener(listener()) -> {ok, pid()} | {error, term()}). +start_listener({Proto, ListenOn, Options}) -> + case start_listener(Proto, ListenOn, Options) of + {ok, _} -> + io:format("Start mqtt:~s listener on ~s successfully.~n", [Proto, format(ListenOn)]); + {error, Reason} -> + io:format(standard_error, "Failed to start mqtt:~s listener on ~s - ~p~n!", + [Proto, format(ListenOn), Reason]) + end. + +%% Start MQTT/TCP listener +-spec(start_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) + -> {ok, pid()} | {error, term()}). +start_listener(tcp, ListenOn, Options) -> + start_mqtt_listener('mqtt:tcp', ListenOn, Options); + +%% Start MQTT/TLS listener +start_listener(Proto, ListenOn, Options) when Proto == ssl; Proto == tls -> + start_mqtt_listener('mqtt:ssl', ListenOn, Options); + +%% Start MQTT/WS listener +start_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws -> + Dispatch = cowboy_router:compile([{'_', [{mqtt_path(Options), emqx_ws_connection, Options}]}]), + start_http_listener(fun cowboy:start_clear/3, 'mqtt:ws', ListenOn, ranch_opts(Options), Dispatch); + +%% Start MQTT/WSS listener +start_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> + Dispatch = cowboy_router:compile([{'_', [{mqtt_path(Options), emqx_ws_connection, Options}]}]), + start_http_listener(fun cowboy:start_tls/3, 'mqtt:wss', ListenOn, ranch_opts(Options), Dispatch). + +start_mqtt_listener(Name, ListenOn, Options) -> + SockOpts = esockd:parse_opt(Options), + esockd:open(Name, ListenOn, merge_default(SockOpts), + {emqx_connection, start_link, [Options -- SockOpts]}). + +start_http_listener(Start, Name, ListenOn, RanchOpts, Dispatch) -> + Start(Name, with_port(ListenOn, RanchOpts), #{env => #{dispatch => Dispatch}}). + +mqtt_path(Options) -> + proplists:get_value(mqtt_path, Options, "/mqtt"). + +ranch_opts(Options) -> + NumAcceptors = proplists:get_value(acceptors, Options, 4), + MaxConnections = proplists:get_value(max_connections, Options, 1024), + TcpOptions = proplists:get_value(tcp_options, Options, []), + RanchOpts = [{num_acceptors, NumAcceptors}, {max_connections, MaxConnections} | TcpOptions], + case proplists:get_value(ssl_options, Options) of + undefined -> RanchOpts; + SslOptions -> RanchOpts ++ SslOptions + end. + +with_port(Port, Opts) when is_integer(Port) -> + [{port, Port}|Opts]; +with_port({Addr, Port}, Opts) -> + [{ip, Addr}, {port, Port}|Opts]. + +%% @doc Restart all listeners +-spec(restart() -> ok). +restart() -> + lists:foreach(fun restart_listener/1, emqx_config:get_env(listeners, [])). + +-spec(restart_listener(listener()) -> any()). +restart_listener({Proto, ListenOn, Options}) -> + restart_listener(Proto, ListenOn, Options). + +-spec(restart_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) -> any()). +restart_listener(tcp, ListenOn, _Options) -> + esockd:reopen('mqtt:tcp', ListenOn); +restart_listener(Proto, ListenOn, _Options) when Proto == ssl; Proto == tls -> + esockd:reopen('mqtt:ssl', ListenOn); +restart_listener(Proto, ListenOn, Options) when Proto == http; Proto == ws -> + cowboy:stop_listener('mqtt:ws'), + start_listener(Proto, ListenOn, Options); +restart_listener(Proto, ListenOn, Options) when Proto == https; Proto == wss -> + cowboy:stop_listener('mqtt:wss'), + start_listener(Proto, ListenOn, Options); +restart_listener(Proto, ListenOn, _Opts) -> + esockd:reopen(Proto, ListenOn). + +%% @doc Stop all listeners. +-spec(stop() -> ok). +stop() -> + lists:foreach(fun stop_listener/1, emqx_config:get_env(listeners, [])). + +-spec(stop_listener(listener()) -> ok | {error, term()}). +stop_listener({Proto, ListenOn, Opts}) -> + case stop_listener(Proto, ListenOn, Opts) of + ok -> + io:format("Stop mqtt:~s listener on ~s successfully.~n", [Proto, format(ListenOn)]); + {error, Reason} -> + io:format(standard_error, "Failed to stop mqtt:~s listener on ~s - ~p~n.", + [Proto, format(ListenOn), Reason]) + end. + +-spec(stop_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) + -> ok | {error, term()}). +stop_listener(tcp, ListenOn, _Opts) -> + esockd:close('mqtt:tcp', ListenOn); +stop_listener(Proto, ListenOn, _Opts) when Proto == ssl; Proto == tls -> + esockd:close('mqtt:ssl', ListenOn); +stop_listener(Proto, _ListenOn, _Opts) when Proto == http; Proto == ws -> + cowboy:stop_listener('mqtt:ws'); +stop_listener(Proto, _ListenOn, _Opts) when Proto == https; Proto == wss -> + cowboy:stop_listener('mqtt:wss'); +stop_listener(Proto, ListenOn, _Opts) -> + esockd:close(Proto, ListenOn). + +merge_default(Options) -> + case lists:keytake(tcp_options, 1, Options) of + {value, {tcp_options, TcpOpts}, Options1} -> + [{tcp_options, emqx_misc:merge_opts(?MQTT_SOCKOPTS, TcpOpts)} | Options1]; + false -> + [{tcp_options, ?MQTT_SOCKOPTS} | Options] + end. + +format(Port) when is_integer(Port) -> + io_lib:format("0.0.0.0:~w", [Port]); +format({Addr, Port}) when is_list(Addr) -> + io_lib:format("~s:~w", [Addr, Port]); +format({Addr, Port}) when is_tuple(Addr) -> + io_lib:format("~s:~w", [esockd_net:ntoab(Addr), Port]). + diff --git a/src/emqttd_bridge.erl b/src/emqx_local_bridge.erl similarity index 55% rename from src/emqttd_bridge.erl rename to src/emqx_local_bridge.erl index 8349eeec1..df2dda686 100644 --- a/src/emqttd_bridge.erl +++ b/src/emqx_local_bridge.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,79 +11,69 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_bridge). +-module(emqx_local_bridge). --behaviour(gen_server2). +-behaviour(gen_server). --author("Feng Lee "). +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). --include("emqttd.hrl"). - --include("emqttd_protocol.hrl"). - --include("emqttd_internal.hrl"). - -%% API Function Exports -export([start_link/5]). -%% gen_server Function Exports --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). -define(PING_DOWN_INTERVAL, 1000). -record(state, {pool, id, node, subtopic, + qos = ?QOS_0, topic_suffix = <<>>, topic_prefix = <<>>, - mqueue :: emqttd_mqueue:mqueue(), + mqueue :: emqx_mqueue:mqueue(), max_queue_len = 10000, ping_down_interval = ?PING_DOWN_INTERVAL, status = up}). --type(option() :: {topic_suffix, binary()} | +-type(option() :: {qos, emqx_mqtt_types:qos()} | + {topic_suffix, binary()} | {topic_prefix, binary()} | {max_queue_len, pos_integer()} | {ping_down_interval, pos_integer()}). -export_type([option/0]). -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - %% @doc Start a bridge --spec(start_link(any(), pos_integer(), atom(), binary(), [option()]) -> - {ok, pid()} | ignore | {error, term()}). +-spec(start_link(term(), pos_integer(), atom(), binary(), [option()]) + -> {ok, pid()} | ignore | {error, term()}). start_link(Pool, Id, Node, Topic, Options) -> - gen_server2:start_link(?MODULE, [Pool, Id, Node, Topic, Options], []). + gen_server:start_link(?MODULE, [Pool, Id, Node, Topic, Options], [{hibernate_after, 5000}]). -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% gen_server callbacks -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ init([Pool, Id, Node, Topic, Options]) -> - ?GPROC_POOL(join, Pool, Id), process_flag(trap_exit, true), + true = gproc_pool:connect_worker(Pool, {Pool, Id}), case net_kernel:connect_node(Node) of - true -> + true -> true = erlang:monitor_node(Node, true), - Share = iolist_to_binary(["$bridge:", atom_to_list(Node), ":", Topic]), - emqttd:subscribe(Topic, self(), [local, {share, Share}, {qos, ?QOS_0}]), + Group = iolist_to_binary(["$bridge:", atom_to_list(Node), ":", Topic]), + emqx_broker:subscribe(Topic, #{share => Group, qos => ?QOS_0}), State = parse_opts(Options, #state{node = Node, subtopic = Topic}), - MQueue = emqttd_mqueue:new(qname(Node, Topic), - [{max_len, State#state.max_queue_len}], - emqttd_alarm:alarm_fun()), - {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}, - hibernate, {backoff, 1000, 1000, 10000}}; - false -> + MQueue = emqx_mqueue:init(#{max_len => State#state.max_queue_len, + store_qos0 => true}), + {ok, State#state{pool = Pool, id = Id, mqueue = MQueue}}; + false -> {stop, {cannot_connect_node, Node}} end. parse_opts([], State) -> State; +parse_opts([{qos, QoS} | Opts], State) -> + parse_opts(Opts, State#state{qos = QoS}); parse_opts([{topic_suffix, Suffix} | Opts], State) -> parse_opts(Opts, State#state{topic_suffix= Suffix}); parse_opts([{topic_prefix, Prefix} | Opts], State) -> @@ -96,48 +85,43 @@ parse_opts([{ping_down_interval, Interval} | Opts], State) -> parse_opts([_Opt | Opts], State) -> parse_opts(Opts, State). -qname(Node, Topic) when is_atom(Node) -> - qname(atom_to_list(Node), Topic); -qname(Node, Topic) -> - iolist_to_binary(["Bridge:", Node, ":", Topic]). - handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). + emqx_logger:error("[Bridge] unexpected call: ~p", [Req]), + {reply, ignored, State}. handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). + emqx_logger:error("[Bridge] unexpected cast: ~p", [Msg]), + {noreply, State}. -handle_info({dispatch, _Topic, Msg}, State = #state{mqueue = MQ, status = down}) -> - {noreply, State#state{mqueue = emqttd_mqueue:in(Msg, MQ)}}; +handle_info({dispatch, _Topic, Msg}, State = #state{mqueue = Q, status = down}) -> + %% TODO: how to drop??? + {_Dropped, NewQ} = emqx_mqueue:in(Msg, Q), + {noreply, State#state{mqueue = NewQ}}; handle_info({dispatch, _Topic, Msg}, State = #state{node = Node, status = up}) -> - rpc:cast(Node, emqttd, publish, [transform(Msg, State)]), - {noreply, State, hibernate}; + emqx_rpc:cast(Node, emqx_broker, publish, [transform(Msg, State)]), + {noreply, State}; handle_info({nodedown, Node}, State = #state{node = Node, ping_down_interval = Interval}) -> - lager:warning("Bridge Node Down: ~p", [Node]), + emqx_logger:warning("[Bridge] node down: ~s", [Node]), erlang:send_after(Interval, self(), ping_down_node), {noreply, State#state{status = down}, hibernate}; handle_info({nodeup, Node}, State = #state{node = Node}) -> %% TODO: Really fast?? - case emqttd:is_running(Node) of - true -> - lager:warning("Bridge Node Up: ~p", [Node]), - {noreply, dequeue(State#state{status = up})}; - false -> - self() ! {nodedown, Node}, - {noreply, State#state{status = down}} + case emqx:is_running(Node) of + true -> emqx_logger:warning("[Bridge] Node up: ~s", [Node]), + {noreply, dequeue(State#state{status = up})}; + false -> self() ! {nodedown, Node}, + {noreply, State#state{status = down}} end; handle_info(ping_down_node, State = #state{node = Node, ping_down_interval = Interval}) -> Self = self(), spawn_link(fun() -> case net_kernel:connect_node(Node) of - true -> %%TODO: this is not right... fixme later - Self ! {nodeup, Node}; - false -> - erlang:send_after(Interval, Self, ping_down_node) + true -> Self ! {nodeup, Node}; + false -> erlang:send_after(Interval, Self, ping_down_node) end end), {noreply, State}; @@ -146,11 +130,11 @@ handle_info({'EXIT', _Pid, normal}, State) -> {noreply, State}; handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). + emqx_logger:error("[Bridge] unexpected info: ~p", [Info]), + {noreply, State}. terminate(_Reason, #state{pool = Pool, id = Id}) -> - ?GPROC_POOL(leave, Pool, Id), - ok. + gproc_pool:disconnect_worker(Pool, {Pool, Id}). code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -160,15 +144,14 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- dequeue(State = #state{mqueue = MQ}) -> - case emqttd_mqueue:out(MQ) of + case emqx_mqueue:out(MQ) of {empty, MQ1} -> State#state{mqueue = MQ1}; {{value, Msg}, MQ1} -> - handle_info({dispatch, Msg#mqtt_message.topic, Msg}, State), + handle_info({dispatch, Msg#message.topic, Msg}, State), dequeue(State#state{mqueue = MQ1}) end. -transform(Msg = #mqtt_message{topic = Topic}, #state{topic_prefix = Prefix, - topic_suffix = Suffix}) -> - Msg#mqtt_message{topic = <>}. +transform(Msg = #message{topic = Topic}, #state{topic_prefix = Prefix, topic_suffix = Suffix}) -> + Msg#message{topic = <>}. diff --git a/include/emqttd_cli.hrl b/src/emqx_local_bridge_sup.erl similarity index 55% rename from include/emqttd_cli.hrl rename to src/emqx_local_bridge_sup.erl index b99038481..fed9f28a7 100644 --- a/include/emqttd_cli.hrl +++ b/src/emqx_local_bridge_sup.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,13 +11,16 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --define(PRINT_MSG(Msg), io:format(Msg)). +-module(emqx_local_bridge_sup). --define(PRINT(Format, Args), io:format(Format, Args)). +-include("emqx.hrl"). --define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~s~n", [Cmd, Descr])). +-export([start_link/3]). --define(USAGE(CmdList), [?PRINT_CMD(Cmd, Descr) || {Cmd, Descr} <- CmdList]). +-spec(start_link(node(), emqx_topic:topic(), [emqx_local_bridge:option()]) + -> {ok, pid()} | {error, term()}). +start_link(Node, Topic, Options) -> + MFA = {emqx_local_bridge, start_link, [Node, Topic, Options]}, + emqx_pool_sup:start_link({bridge, Node, Topic}, random, MFA). diff --git a/src/emqttd_bridge_sup_sup.erl b/src/emqx_local_bridge_sup_sup.erl similarity index 51% rename from src/emqttd_bridge_sup_sup.erl rename to src/emqx_local_bridge_sup_sup.erl index fe5c33428..0483552b2 100644 --- a/src/emqttd_bridge_sup_sup.erl +++ b/src/emqx_local_bridge_sup_sup.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,16 +11,17 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_bridge_sup_sup). +-module(emqx_local_bridge_sup_sup). -behavior(supervisor). --author("Feng Lee "). +-include("emqx.hrl"). --export([start_link/0, bridges/0, start_bridge/2, start_bridge/3, stop_bridge/2]). +-export([start_link/0, bridges/0]). +-export([start_bridge/2, start_bridge/3, stop_bridge/2]). +%% Supervisor callbacks -export([init/1]). -define(CHILD_ID(Node, Topic), {bridge_sup, Node, Topic}). @@ -29,47 +29,46 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - %% @doc List all bridges --spec(bridges() -> [{node(), binary(), pid()}]). +-spec(bridges() -> [{node(), emqx_topic:topic(), pid()}]). bridges() -> [{Node, Topic, Pid} || {?CHILD_ID(Node, Topic), Pid, supervisor, _} - <- supervisor:which_children(?MODULE)]. + <- supervisor:which_children(?MODULE)]. %% @doc Start a bridge --spec(start_bridge(atom(), binary()) -> {ok, pid()} | {error, term()}). -start_bridge(Node, Topic) when is_atom(Node) andalso is_binary(Topic) -> +-spec(start_bridge(node(), emqx_topic:topic()) -> {ok, pid()} | {error, term()}). +start_bridge(Node, Topic) when is_atom(Node), is_binary(Topic) -> start_bridge(Node, Topic, []). --spec(start_bridge(atom(), binary(), [emqttd_bridge:option()]) -> {ok, pid()} | {error, term()}). +-spec(start_bridge(node(), emqx_topic:topic(), [emqx_bridge:option()]) + -> {ok, pid()} | {error, term()}). start_bridge(Node, _Topic, _Options) when Node =:= node() -> {error, bridge_to_self}; -start_bridge(Node, Topic, Options) when is_atom(Node) andalso is_binary(Topic) -> - {ok, BridgeEnv} = emqttd:env(bridge), - Options1 = emqttd_misc:merge_opts(BridgeEnv, Options), +start_bridge(Node, Topic, Options) when is_atom(Node), is_binary(Topic) -> + Options1 = emqx_misc:merge_opts(emqx_config:get_env(bridge, []), Options), supervisor:start_child(?MODULE, bridge_spec(Node, Topic, Options1)). %% @doc Stop a bridge --spec(stop_bridge(atom(), binary()) -> {ok, pid()} | ok). -stop_bridge(Node, Topic) when is_atom(Node) andalso is_binary(Topic) -> +-spec(stop_bridge(node(), emqx_topic:topic()) -> ok | {error, term()}). +stop_bridge(Node, Topic) when is_atom(Node), is_binary(Topic) -> ChildId = ?CHILD_ID(Node, Topic), case supervisor:terminate_child(?MODULE, ChildId) of ok -> supervisor:delete_child(?MODULE, ChildId); Error -> Error end. -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Supervisor callbacks -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ init([]) -> - {ok, {{one_for_one, 10, 100}, []}}. + {ok, {{one_for_one, 10, 3600}, []}}. bridge_spec(Node, Topic, Options) -> - {?CHILD_ID(Node, Topic), - {emqttd_bridge_sup, start_link, [Node, Topic, Options]}, - permanent, infinity, supervisor, [emqttd_bridge_sup]}. + #{id => ?CHILD_ID(Node, Topic), + start => {emqx_local_bridge_sup, start_link, [Node, Topic, Options]}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_local_bridge_sup]}. diff --git a/src/emqx_logger.erl b/src/emqx_logger.erl new file mode 100644 index 000000000..b3c9e9d54 --- /dev/null +++ b/src/emqx_logger.erl @@ -0,0 +1,110 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_logger). + +-compile({no_auto_import,[error/1]}). + +-export([debug/1, debug/2, debug/3]). +-export([info/1, info/2, info/3]). +-export([warning/1, warning/2, warning/3]). +-export([error/1, error/2, error/3]). +-export([critical/1, critical/2, critical/3]). + +-export([set_metadata_peername/1, set_metadata_client_id/1]). +-export([set_proc_metadata/1]). + +-export([get_primary_log_level/0, set_primary_log_level/1]). +-export([get_log_handlers/0, get_log_handler/1, set_log_handler_level/2]). + +debug(Msg) -> + logger:debug(Msg). +debug(Format, Args) -> + logger:debug(Format, Args). +debug(Metadata, Format, Args) when is_map(Metadata) -> + logger:debug(Format, Args, Metadata). + +info(Msg) -> + logger:info(Msg). +info(Format, Args) -> + logger:info(Format, Args). +info(Metadata, Format, Args) when is_map(Metadata) -> + logger:info(Format, Args, Metadata). + +warning(Msg) -> + logger:warning(Msg). +warning(Format, Args) -> + logger:warning(Format, Args). +warning(Metadata, Format, Args) when is_map(Metadata) -> + logger:warning(Format, Args, Metadata). + +error(Msg) -> + logger:error(Msg). +error(Format, Args) -> + logger:error(Format, Args). +error(Metadata, Format, Args) when is_map(Metadata) -> + logger:error(Format, Args, Metadata). + +critical(Msg) -> + logger:critical(Msg). +critical(Format, Args) -> + logger:critical(Format, Args). +critical(Metadata, Format, Args) when is_map(Metadata) -> + logger:critical(Format, Args, Metadata). + +set_metadata_client_id(ClientId) -> + set_proc_metadata(#{client_id => ClientId}). + +set_metadata_peername(Peername) -> + set_proc_metadata(#{peername => Peername}). + +set_proc_metadata(Meta) -> + logger:update_process_metadata(Meta). + +get_primary_log_level() -> + #{level := Level} = logger:get_primary_config(), + Level. + +set_primary_log_level(Level) -> + logger:set_primary_config(level, Level). + +get_log_handlers() -> + lists:map(fun log_hanlder_info/1, logger:get_handler_config()). + +get_log_handler(HandlerId) -> + {ok, Conf} = logger:get_handler_config(HandlerId), + log_hanlder_info(Conf). + +set_log_handler_level(HandlerId, Level) -> + logger:set_handler_config(HandlerId, level, Level). + +%%======================== +%% Internal Functions +%%======================== +log_hanlder_info(#{id := Id, level := Level, module := logger_std_h, + config := #{type := Type}}) when Type =:= standard_io; + Type =:= standard_error -> + {Id, Level, console}; +log_hanlder_info(#{id := Id, level := Level, module := logger_std_h, + config := #{type := Type}}) -> + case Type of + {file, Filename} -> {Id, Level, Filename}; + {file, Filename, _Opts} -> {Id, Level, Filename}; + _ -> {Id, Level, unknown} + end; +log_hanlder_info(#{id := Id, level := Level, module := logger_disk_log_h, + config := #{file := Filename}}) -> + {Id, Level, Filename}; +log_hanlder_info(#{id := Id, level := Level, module := _OtherModule}) -> + {Id, Level, unknown}. \ No newline at end of file diff --git a/src/emqx_logger_formatter.erl b/src/emqx_logger_formatter.erl new file mode 100644 index 000000000..3c5d9fc16 --- /dev/null +++ b/src/emqx_logger_formatter.erl @@ -0,0 +1,360 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2017-2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%% This file is copied from lib/kernel/src/logger_formatter.erl, and +%% modified for a more concise time format other than the default RFC3339. + +-module(emqx_logger_formatter). + +-export([format/2]). +-export([check_config/1]). + +-define(DEFAULT_FORMAT_TEMPLATE_SINGLE, [time," ",level,": ",msg,"\n"]). + +-define(FormatP, "~0tp"). + +-define(IS_STRING(String), + (is_list(String) orelse is_binary(String))). + +%%%----------------------------------------------------------------- +%%% Types +-type config() :: #{chars_limit => pos_integer() | unlimited, + depth => pos_integer() | unlimited, + max_size => pos_integer() | unlimited, + report_cb => logger:report_cb(), + quit => template()}. +-type template() :: [metakey() | {metakey(),template(),template()} | string()]. +-type metakey() :: atom() | [atom()]. + +%%%----------------------------------------------------------------- +%%% API +-spec format(LogEvent,Config) -> unicode:chardata() when + LogEvent :: logger:log_event(), + Config :: config(). +format(#{level:=Level,msg:=Msg0,meta:=Meta},Config0) + when is_map(Config0) -> + Config = add_default_config(Config0), + Template = maps:get(template,Config), + {BT,AT0} = lists:splitwith(fun(msg) -> false; (_) -> true end, Template), + {DoMsg,AT} = + case AT0 of + [msg|Rest] -> {true,Rest}; + _ ->{false,AT0} + end, + B = do_format(Level,Meta,BT,Config), + A = do_format(Level,Meta,AT,Config), + MsgStr = + if DoMsg -> + Config1 = + case maps:get(chars_limit,Config) of + unlimited -> + Config; + Size0 -> + Size = + case Size0 - string:length([B,A]) of + S when S>=0 -> S; + _ -> 0 + end, + Config#{chars_limit=>Size} + end, + string:trim(format_msg(Msg0,Meta,Config1)); + true -> + "" + end, + truncate([B,MsgStr,A],maps:get(max_size,Config)). + +do_format(Level,Data,[level|Format],Config) -> + [to_string(level,Level,Config)|do_format(Level,Data,Format,Config)]; +do_format(Level,Data,[{Key,IfExist,Else}|Format],Config) -> + String = + case value(Key,Data) of + {ok,Value} -> do_format(Level,Data#{Key=>Value},IfExist,Config); + error -> do_format(Level,Data,Else,Config) + end, + [String|do_format(Level,Data,Format,Config)]; +do_format(Level,Data,[Key|Format],Config) + when is_atom(Key) orelse + (is_list(Key) andalso is_atom(hd(Key))) -> + String = + case value(Key,Data) of + {ok,Value} -> to_string(Key,Value,Config); + error -> "" + end, + [String|do_format(Level,Data,Format,Config)]; +do_format(Level,Data,[Str|Format],Config) -> + [Str|do_format(Level,Data,Format,Config)]; +do_format(_Level,_Data,[],_Config) -> + []. + +value(Key,Meta) when is_map_key(Key,Meta) -> + {ok,maps:get(Key,Meta)}; +value([Key|Keys],Meta) when is_map_key(Key,Meta) -> + value(Keys,maps:get(Key,Meta)); +value([],Value) -> + {ok,Value}; +value(_,_) -> + error. + +to_string(time,Time,Config) -> + format_time(Time,Config); +to_string(mfa,MFA,Config) -> + format_mfa(MFA,Config); +to_string(_,Value,Config) -> + to_string(Value,Config). + +to_string(X,_) when is_atom(X) -> + atom_to_list(X); +to_string(X,_) when is_integer(X) -> + integer_to_list(X); +to_string(X,_) when is_pid(X) -> + pid_to_list(X); +to_string(X,_) when is_reference(X) -> + ref_to_list(X); +to_string(X,_) when is_list(X) -> + case printable_list(lists:flatten(X)) of + true -> X; + _ -> io_lib:format(?FormatP,[X]) + end; +to_string(X,_) -> + io_lib:format("~s",[X]). + +printable_list([]) -> + false; +printable_list(X) -> + io_lib:printable_list(X). + +format_msg({string,Chardata},Meta,Config) -> + format_msg({"~ts",[Chardata]},Meta,Config); +format_msg({report,_}=Msg,Meta,#{report_cb:=Fun}=Config) + when is_function(Fun,1); is_function(Fun,2) -> + format_msg(Msg,Meta#{report_cb=>Fun},maps:remove(report_cb,Config)); +format_msg({report,Report},#{report_cb:=Fun}=Meta,Config) when is_function(Fun,1) -> + try Fun(Report) of + {Format,Args} when is_list(Format), is_list(Args) -> + format_msg({Format,Args},maps:remove(report_cb,Meta),Config); + Other -> + format_msg({"REPORT_CB/1 ERROR: ~0tp; Returned: ~0tp", + [Report,Other]},Meta,Config) + catch C:R:S -> + format_msg({"REPORT_CB/1 CRASH: ~0tp; Reason: ~0tp", + [Report,{C,R,logger:filter_stacktrace(?MODULE,S)}]},Meta,Config) + end; +format_msg({report,Report},#{report_cb:=Fun}=Meta,Config) when is_function(Fun,2) -> + try Fun(Report,maps:with([depth,chars_limit,single_line],Config)) of + Chardata when ?IS_STRING(Chardata) -> + try chardata_to_list(Chardata) % already size limited by report_cb + catch _:_ -> + format_msg({"REPORT_CB/2 ERROR: ~0tp; Returned: ~0tp", + [Report,Chardata]},Meta,Config) + end; + Other -> + format_msg({"REPORT_CB/2 ERROR: ~0tp; Returned: ~0tp", + [Report,Other]},Meta,Config) + catch C:R:S -> + format_msg({"REPORT_CB/2 CRASH: ~0tp; Reason: ~0tp", + [Report,{C,R,logger:filter_stacktrace(?MODULE,S)}]}, + Meta,Config) + end; +format_msg({report,Report},Meta,Config) -> + format_msg({report,Report}, + Meta#{report_cb=>fun logger:format_report/1}, + Config); +format_msg(Msg,_Meta,#{depth:=Depth,chars_limit:=CharsLimit}) -> + Opts = chars_limit_to_opts(CharsLimit), + do_format_msg(Msg, Depth, Opts). + +chars_limit_to_opts(unlimited) -> []; +chars_limit_to_opts(CharsLimit) -> [{chars_limit,CharsLimit}]. + +do_format_msg({Format0,Args},Depth,Opts) -> + try + Format1 = io_lib:scan_format(Format0, Args), + Format = reformat(Format1, Depth), + io_lib:build_text(Format,Opts) + catch C:R:S -> + FormatError = "FORMAT ERROR: ~0tp - ~0tp", + case Format0 of + FormatError -> + %% already been here - avoid failing cyclically + erlang:raise(C,R,S); + _ -> + format_msg({FormatError,[Format0,Args]},Depth,Opts) + end + end. + +reformat(Format,unlimited) -> + Format; +reformat([#{control_char:=C}=M|T], Depth) when C =:= $p -> + [limit_depth(M#{width => 0}, Depth)|reformat(T, Depth)]; +reformat([#{control_char:=C}=M|T], Depth) when C =:= $P -> + [M#{width => 0}|reformat(T, Depth)]; +reformat([#{control_char:=C}=M|T], Depth) when C =:= $p; C =:= $w -> + [limit_depth(M, Depth)|reformat(T, Depth)]; +reformat([H|T], Depth) -> + [H|reformat(T, Depth)]; +reformat([], _) -> + []. + +limit_depth(M0, unlimited) -> + M0; +limit_depth(#{control_char:=C0, args:=Args}=M0, Depth) -> + C = C0 - ($a - $A), %To uppercase. + M0#{control_char:=C,args:=Args++[Depth]}. + +chardata_to_list(Chardata) -> + case unicode:characters_to_list(Chardata,unicode) of + List when is_list(List) -> + List; + Error -> + throw(Error) + end. + +truncate(String,unlimited) -> + String; +truncate(String,Size) -> + Length = string:length(String), + if Length>Size -> + case lists:reverse(lists:flatten(String)) of + [$\n|_] -> + string:slice(String,0,Size-4)++"...\n"; + _ -> + string:slice(String,0,Size-3)++"..." + end; + true -> + String + end. + +%% Convert microseconds-timestamp into local datatime string in milliseconds +format_time(SysTime,#{}) + when is_integer(SysTime) -> + Ms = SysTime rem 1000000 div 1000, + {Date, _Time = {H, Mi, S}} = calendar:system_time_to_local_time(SysTime, microsecond), + format_time({Date, {H, Mi, S, Ms}}). +format_time({{Y, M, D}, {H, Mi, S, Ms}}) -> + io_lib:format("~b-~2..0b-~2..0b ~2..0b:~2..0b:~2..0b.~3..0b", [Y, M, D, H, Mi, S, Ms]); +format_time({{Y, M, D}, {H, Mi, S}}) -> + io_lib:format("~b-~2..0b-~2..0b ~2..0b:~2..0b:~2..0b", [Y, M, D, H, Mi, S]). + +format_mfa({M,F,A},_) when is_atom(M), is_atom(F), is_integer(A) -> + atom_to_list(M)++":"++atom_to_list(F)++"/"++integer_to_list(A); +format_mfa({M,F,A},Config) when is_atom(M), is_atom(F), is_list(A) -> + format_mfa({M,F,length(A)},Config); +format_mfa(MFA,Config) -> + to_string(MFA,Config). + +%% Ensure that all valid configuration parameters exist in the final +%% configuration map +add_default_config(Config0) -> + Default = + #{chars_limit=>unlimited, + error_logger_notice_header=>info}, + MaxSize = get_max_size(maps:get(max_size,Config0,undefined)), + Depth = get_depth(maps:get(depth,Config0,undefined)), + add_default_template(maps:merge(Default,Config0#{max_size=>MaxSize, + depth=>Depth})). + +add_default_template(#{template:=_}=Config) -> + Config; +add_default_template(Config) -> + Config#{template=>?DEFAULT_FORMAT_TEMPLATE_SINGLE}. + +get_max_size(undefined) -> + unlimited; +get_max_size(S) -> + max(10,S). + +get_depth(undefined) -> + error_logger:get_format_depth(); +get_depth(S) -> + max(5,S). + +-spec check_config(Config) -> ok | {error,term()} when + Config :: config(). +check_config(Config) when is_map(Config) -> + do_check_config(maps:to_list(Config)); +check_config(Config) -> + {error,{invalid_formatter_config,?MODULE,Config}}. + +do_check_config([{Type,L}|Config]) when Type == chars_limit; + Type == depth; + Type == max_size -> + case check_limit(L) of + ok -> do_check_config(Config); + error -> {error,{invalid_formatter_config,?MODULE,{Type,L}}} + end; +do_check_config([{error_logger_notice_header,ELNH}|Config]) when ELNH == info; + ELNH == notice -> + do_check_config(Config); +do_check_config([{report_cb,RCB}|Config]) when is_function(RCB,1); + is_function(RCB,2) -> + do_check_config(Config); +do_check_config([{template,T}|Config]) -> + case check_template(T) of + ok -> do_check_config(Config); + error -> {error,{invalid_formatter_template,?MODULE,T}} + end; + +do_check_config([C|_]) -> + {error,{invalid_formatter_config,?MODULE,C}}; +do_check_config([]) -> + ok. + +check_limit(L) when is_integer(L), L>0 -> + ok; +check_limit(unlimited) -> + ok; +check_limit(_) -> + error. + +check_template([Key|T]) when is_atom(Key) -> + check_template(T); +check_template([Key|T]) when is_list(Key), is_atom(hd(Key)) -> + case lists:all(fun(X) when is_atom(X) -> true; + (_) -> false + end, + Key) of + true -> + check_template(T); + false -> + error + end; +check_template([{Key,IfExist,Else}|T]) + when is_atom(Key) orelse + (is_list(Key) andalso is_atom(hd(Key))) -> + case check_template(IfExist) of + ok -> + case check_template(Else) of + ok -> + check_template(T); + error -> + error + end; + error -> + error + end; +check_template([Str|T]) when is_list(Str) -> + case io_lib:printable_unicode_list(Str) of + true -> check_template(T); + false -> error + end; +check_template([]) -> + ok; +check_template(_) -> + error. diff --git a/src/emqx_message.erl b/src/emqx_message.erl new file mode 100644 index 000000000..a4bbb378c --- /dev/null +++ b/src/emqx_message.erl @@ -0,0 +1,129 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_message). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-export([make/2, make/3, make/4]). +-export([set_flags/2]). +-export([get_flag/2, get_flag/3, set_flag/2, set_flag/3, unset_flag/2]). +-export([set_headers/2]). +-export([get_header/2, get_header/3, set_header/3]). +-export([is_expired/1, update_expiry/1]). +-export([remove_topic_alias/1]). +-export([format/1]). + +-type(flag() :: atom()). + +-spec(make(emqx_topic:topic(), emqx_types:payload()) -> emqx_types:message()). +make(Topic, Payload) -> + make(undefined, Topic, Payload). + +-spec(make(atom() | emqx_types:client_id(), emqx_topic:topic(), emqx_types:payload()) + -> emqx_types:message()). +make(From, Topic, Payload) -> + make(From, ?QOS_0, Topic, Payload). + +-spec(make(atom() | emqx_types:client_id(), emqx_mqtt_types:qos(), + emqx_topic:topic(), emqx_types:payload()) -> emqx_types:message()). +make(From, QoS, Topic, Payload) -> + #message{id = msgid(QoS), + qos = QoS, + from = From, + flags = #{dup => false}, + topic = Topic, + payload = Payload, + timestamp = os:timestamp()}. + +msgid(?QOS_0) -> undefined; +msgid(_QoS) -> emqx_guid:gen(). + +set_flags(Flags, Msg = #message{flags = undefined}) when is_map(Flags) -> + Msg#message{flags = Flags}; +set_flags(New, Msg = #message{flags = Old}) when is_map(New) -> + Msg#message{flags = maps:merge(Old, New)}. + +get_flag(Flag, Msg) -> + get_flag(Flag, Msg, false). +get_flag(Flag, #message{flags = Flags}, Default) -> + maps:get(Flag, Flags, Default). + +-spec(set_flag(flag(), emqx_types:message()) -> emqx_types:message()). +set_flag(Flag, Msg = #message{flags = undefined}) when is_atom(Flag) -> + Msg#message{flags = #{Flag => true}}; +set_flag(Flag, Msg = #message{flags = Flags}) when is_atom(Flag) -> + Msg#message{flags = maps:put(Flag, true, Flags)}. + +-spec(set_flag(flag(), boolean() | integer(), emqx_types:message()) + -> emqx_types:message()). +set_flag(Flag, Val, Msg = #message{flags = undefined}) when is_atom(Flag) -> + Msg#message{flags = #{Flag => Val}}; +set_flag(Flag, Val, Msg = #message{flags = Flags}) when is_atom(Flag) -> + Msg#message{flags = maps:put(Flag, Val, Flags)}. + +-spec(unset_flag(flag(), emqx_types:message()) -> emqx_types:message()). +unset_flag(Flag, Msg = #message{flags = Flags}) -> + Msg#message{flags = maps:remove(Flag, Flags)}. + +set_headers(Headers, Msg = #message{headers = undefined}) when is_map(Headers) -> + Msg#message{headers = Headers}; +set_headers(New, Msg = #message{headers = Old}) when is_map(New) -> + Msg#message{headers = maps:merge(Old, New)}; +set_headers(_, Msg) -> + Msg. + +get_header(Hdr, Msg) -> + get_header(Hdr, Msg, undefined). +get_header(Hdr, #message{headers = Headers}, Default) -> + maps:get(Hdr, Headers, Default). + +set_header(Hdr, Val, Msg = #message{headers = undefined}) -> + Msg#message{headers = #{Hdr => Val}}; +set_header(Hdr, Val, Msg = #message{headers = Headers}) -> + Msg#message{headers = maps:put(Hdr, Val, Headers)}. + +-spec(is_expired(emqx_types:message()) -> boolean()). +is_expired(#message{headers = #{'Message-Expiry-Interval' := Interval}, timestamp = CreatedAt}) -> + elapsed(CreatedAt) > timer:seconds(Interval); +is_expired(_Msg) -> + false. + +update_expiry(Msg = #message{headers = #{'Message-Expiry-Interval' := Interval}, timestamp = CreatedAt}) -> + case elapsed(CreatedAt) of + Elapsed when Elapsed > 0 -> + set_header('Message-Expiry-Interval', max(1, Interval - (Elapsed div 1000)), Msg); + _ -> Msg + end; + +update_expiry(Msg) -> Msg. + +remove_topic_alias(Msg = #message{headers = Headers}) -> + Msg#message{headers = maps:remove('Topic-Alias', Headers)}. + +%% MilliSeconds +elapsed(Since) -> + max(0, timer:now_diff(os:timestamp(), Since) div 1000). + +format(#message{id = Id, qos = QoS, topic = Topic, from = From, flags = Flags, headers = Headers}) -> + io_lib:format("Message(Id=~s, QoS=~w, Topic=~s, From=~p, Flags=~s, Headers=~s)", + [Id, QoS, Topic, From, format(flags, Flags), format(headers, Headers)]). + +format(_, undefined) -> + ""; +format(flags, Flags) -> + io_lib:format("~p", [[Flag || {Flag, true} <- maps:to_list(Flags)]]); +format(headers, Headers) -> + io_lib:format("~p", [Headers]). diff --git a/src/emqx_metrics.erl b/src/emqx_metrics.erl new file mode 100644 index 000000000..b4b3a1307 --- /dev/null +++ b/src/emqx_metrics.erl @@ -0,0 +1,309 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_metrics). + +-include("emqx_mqtt.hrl"). + +-export([start_link/0]). +-export([new/1, all/0]). +-export([val/1, inc/1, inc/2, inc/3, dec/2, dec/3, set/2]). +-export([trans/2, trans/3, trans/4, commit/0]). +%% Received/sent metrics +-export([received/1, sent/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +%% Bytes sent and received of Broker +-define(BYTES_METRICS, [ + {counter, 'bytes/received'}, % Total bytes received + {counter, 'bytes/sent'} % Total bytes sent +]). + +%% Packets sent and received of broker +-define(PACKET_METRICS, [ + {counter, 'packets/received'}, % All Packets received + {counter, 'packets/sent'}, % All Packets sent + {counter, 'packets/connect'}, % CONNECT Packets received + {counter, 'packets/connack'}, % CONNACK Packets sent + {counter, 'packets/publish/received'}, % PUBLISH packets received + {counter, 'packets/publish/sent'}, % PUBLISH packets sent + {counter, 'packets/puback/received'}, % PUBACK packets received + {counter, 'packets/puback/sent'}, % PUBACK packets sent + {counter, 'packets/puback/missed'}, % PUBACK packets missed + {counter, 'packets/pubrec/received'}, % PUBREC packets received + {counter, 'packets/pubrec/sent'}, % PUBREC packets sent + {counter, 'packets/pubrec/missed'}, % PUBREC packets missed + {counter, 'packets/pubrel/received'}, % PUBREL packets received + {counter, 'packets/pubrel/sent'}, % PUBREL packets sent + {counter, 'packets/pubrel/missed'}, % PUBREL packets missed + {counter, 'packets/pubcomp/received'}, % PUBCOMP packets received + {counter, 'packets/pubcomp/sent'}, % PUBCOMP packets sent + {counter, 'packets/pubcomp/missed'}, % PUBCOMP packets missed + {counter, 'packets/subscribe'}, % SUBSCRIBE Packets received + {counter, 'packets/suback'}, % SUBACK packets sent + {counter, 'packets/unsubscribe'}, % UNSUBSCRIBE Packets received + {counter, 'packets/unsuback'}, % UNSUBACK Packets sent + {counter, 'packets/pingreq'}, % PINGREQ packets received + {counter, 'packets/pingresp'}, % PINGRESP Packets sent + {counter, 'packets/disconnect/received'}, % DISCONNECT Packets received + {counter, 'packets/disconnect/sent'}, % DISCONNECT Packets sent + {counter, 'packets/auth'} % Auth Packets received +]). + +%% Messages sent and received of broker +-define(MESSAGE_METRICS, [ + {counter, 'messages/received'}, % All Messages received + {counter, 'messages/sent'}, % All Messages sent + {counter, 'messages/qos0/received'}, % QoS0 Messages received + {counter, 'messages/qos0/sent'}, % QoS0 Messages sent + {counter, 'messages/qos1/received'}, % QoS1 Messages received + {counter, 'messages/qos1/sent'}, % QoS1 Messages sent + {counter, 'messages/qos2/received'}, % QoS2 Messages received + {counter, 'messages/qos2/expired'}, % QoS2 Messages expired + {counter, 'messages/qos2/sent'}, % QoS2 Messages sent + {counter, 'messages/qos2/dropped'}, % QoS2 Messages dropped + {gauge, 'messages/retained'}, % Messagea retained + {counter, 'messages/dropped'}, % Messages dropped + {counter, 'messages/expired'}, % Messages expired + {counter, 'messages/forward'} % Messages forward +]). + +-define(TAB, ?MODULE). +-define(SERVER, ?MODULE). + +%% @doc Start the metrics server. +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +%%------------------------------------------------------------------------------ +%% Metrics API +%%------------------------------------------------------------------------------ + +new({gauge, Name}) -> + ets:insert(?TAB, {{Name, 0}, 0}); + +new({counter, Name}) -> + Schedulers = lists:seq(1, emqx_vm:schedulers()), + ets:insert(?TAB, [{{Name, I}, 0} || I <- Schedulers]). + +%% @doc Get all metrics +-spec(all() -> [{atom(), non_neg_integer()}]). +all() -> + maps:to_list( + ets:foldl( + fun({{Metric, _N}, Val}, Map) -> + case maps:find(Metric, Map) of + {ok, Count} -> maps:put(Metric, Count+Val, Map); + error -> maps:put(Metric, Val, Map) + end + end, #{}, ?TAB)). + +%% @doc Get metric value +-spec(val(atom()) -> non_neg_integer()). +val(Metric) -> + lists:sum(ets:select(?TAB, [{{{Metric, '_'}, '$1'}, [], ['$1']}])). + +%% @doc Increase counter +-spec(inc(atom()) -> non_neg_integer()). +inc(Metric) -> + inc(counter, Metric, 1). + +%% @doc Increase metric value +-spec(inc({counter | gauge, atom()} | atom(), pos_integer()) -> non_neg_integer()). +inc({gauge, Metric}, Val) -> + inc(gauge, Metric, Val); +inc({counter, Metric}, Val) -> + inc(counter, Metric, Val); +inc(Metric, Val) when is_atom(Metric) -> + inc(counter, Metric, Val). + +%% @doc Increase metric value +-spec(inc(counter | gauge, atom(), pos_integer()) -> pos_integer()). +inc(Type, Metric, Val) -> + update_counter(key(Type, Metric), {2, Val}). + +%% @doc Decrease metric value +-spec(dec(gauge, atom()) -> integer()). +dec(gauge, Metric) -> + dec(gauge, Metric, 1). + +%% @doc Decrease metric value +-spec(dec(gauge, atom(), pos_integer()) -> integer()). +dec(gauge, Metric, Val) -> + update_counter(key(gauge, Metric), {2, -Val}). + +%% @doc Set metric value +set(Metric, Val) when is_atom(Metric) -> + set(gauge, Metric, Val). +set(gauge, Metric, Val) -> + ets:insert(?TAB, {key(gauge, Metric), Val}). + +trans(inc, Metric) -> + trans(inc, {counter, Metric}, 1). + +trans(Opt, {gauge, Metric}, Val) -> + trans(Opt, gauge, Metric, Val); +trans(inc, {counter, Metric}, Val) -> + trans(inc, counter, Metric, Val); +trans(inc, Metric, Val) when is_atom(Metric) -> + trans(inc, counter, Metric, Val); +trans(dec, gauge, Metric) -> + trans(dec, gauge, Metric, 1). + +trans(inc, Type, Metric, Val) -> + hold(Type, Metric, Val); +trans(dec, gauge, Metric, Val) -> + hold(gauge, Metric, -Val). + +hold(Type, Metric, Val) when Type =:= counter orelse Type =:= gauge -> + put('$metrics', case get('$metrics') of + undefined -> + #{{Type, Metric} => Val}; + Metrics -> + maps:update_with({Type, Metric}, fun(Cnt) -> Cnt + Val end, Val, Metrics) + end). + +commit() -> + case get('$metrics') of + undefined -> ok; + Metrics -> + maps:fold(fun({Type, Metric}, Val, _Acc) -> + update_counter(key(Type, Metric), {2, Val}) + end, 0, Metrics), + erase('$metrics') + end. + +%% @doc Metric key +key(gauge, Metric) -> + {Metric, 0}; +key(counter, Metric) -> + {Metric, erlang:system_info(scheduler_id)}. + +update_counter(Key, UpOp) -> + ets:update_counter(?TAB, Key, UpOp). + +%%----------------------------------------------------------------------------- +%% Received/Sent metrics +%%----------------------------------------------------------------------------- + +%% @doc Count packets received. +-spec(received(emqx_mqtt_types:packet()) -> ok). +received(Packet) -> + inc('packets/received'), + received1(Packet). +received1(?PUBLISH_PACKET(QoS, _PktId)) -> + inc('packets/publish/received'), + inc('messages/received'), + qos_received(QoS); +received1(?PACKET(Type)) -> + received2(Type). +received2(?CONNECT) -> + inc('packets/connect'); +received2(?PUBACK) -> + inc('packets/puback/received'); +received2(?PUBREC) -> + inc('packets/pubrec/received'); +received2(?PUBREL) -> + inc('packets/pubrel/received'); +received2(?PUBCOMP) -> + inc('packets/pubcomp/received'); +received2(?SUBSCRIBE) -> + inc('packets/subscribe'); +received2(?UNSUBSCRIBE) -> + inc('packets/unsubscribe'); +received2(?PINGREQ) -> + inc('packets/pingreq'); +received2(?DISCONNECT) -> + inc('packets/disconnect/received'); +received2(_) -> + ignore. +qos_received(?QOS_0) -> + inc('messages/qos0/received'); +qos_received(?QOS_1) -> + inc('messages/qos1/received'); +qos_received(?QOS_2) -> + inc('messages/qos2/received'). + +%% @doc Count packets received. Will not count $SYS PUBLISH. +-spec(sent(emqx_mqtt_types:packet()) -> ignore | non_neg_integer()). +sent(?PUBLISH_PACKET(_QoS, <<"$SYS/", _/binary>>, _, _)) -> + ignore; +sent(Packet) -> + inc('packets/sent'), + sent1(Packet). +sent1(?PUBLISH_PACKET(QoS, _PktId)) -> + inc('packets/publish/sent'), + inc('messages/sent'), + qos_sent(QoS); +sent1(?PACKET(Type)) -> + sent2(Type). +sent2(?CONNACK) -> + inc('packets/connack'); +sent2(?PUBACK) -> + inc('packets/puback/sent'); +sent2(?PUBREC) -> + inc('packets/pubrec/sent'); +sent2(?PUBREL) -> + inc('packets/pubrel/sent'); +sent2(?PUBCOMP) -> + inc('packets/pubcomp/sent'); +sent2(?SUBACK) -> + inc('packets/suback'); +sent2(?UNSUBACK) -> + inc('packets/unsuback'); +sent2(?PINGRESP) -> + inc('packets/pingresp'); +sent2(?DISCONNECT) -> + inc('packets/disconnect/sent'); +sent2(_Type) -> + ignore. +qos_sent(?QOS_0) -> + inc('messages/qos0/sent'); +qos_sent(?QOS_1) -> + inc('messages/qos1/sent'); +qos_sent(?QOS_2) -> + inc('messages/qos2/sent'). + +%%----------------------------------------------------------------------------- +%% gen_server callbacks +%%----------------------------------------------------------------------------- + +init([]) -> + % Create metrics table + ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), + lists:foreach(fun new/1, ?BYTES_METRICS ++ ?PACKET_METRICS ++ ?MESSAGE_METRICS), + {ok, #{}, hibernate}. + +handle_call(Req, _From, State) -> + emqx_logger:error("[Metrics] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[Metrics] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + emqx_logger:error("[Metrics] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{}) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + diff --git a/src/emqx_misc.erl b/src/emqx_misc.erl new file mode 100644 index 000000000..38bc1c2b0 --- /dev/null +++ b/src/emqx_misc.erl @@ -0,0 +1,128 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_misc). + +-export([merge_opts/2, start_timer/2, start_timer/3, cancel_timer/1, + proc_name/2, proc_stats/0, proc_stats/1]). + +-export([init_proc_mng_policy/1, conn_proc_mng_policy/1]). + +-export([drain_down/1]). + +%% @doc Merge options +-spec(merge_opts(list(), list()) -> list()). +merge_opts(Defaults, Options) -> + lists:foldl( + fun({Opt, Val}, Acc) -> + lists:keystore(Opt, 1, Acc, {Opt, Val}); + (Opt, Acc) -> + lists:usort([Opt | Acc]) + end, Defaults, Options). + +-spec(start_timer(integer(), term()) -> reference()). +start_timer(Interval, Msg) -> + start_timer(Interval, self(), Msg). + +-spec(start_timer(integer(), pid() | atom(), term()) -> reference()). +start_timer(Interval, Dest, Msg) -> + erlang:start_timer(Interval, Dest, Msg). + +-spec(cancel_timer(undefined | reference()) -> ok). +cancel_timer(Timer) when is_reference(Timer) -> + case erlang:cancel_timer(Timer) of + false -> + receive {timeout, Timer, _} -> ok after 0 -> ok end; + _ -> ok + end; +cancel_timer(_) -> ok. + +-spec(proc_name(atom(), pos_integer()) -> atom()). +proc_name(Mod, Id) -> + list_to_atom(lists:concat([Mod, "_", Id])). + +-spec(proc_stats() -> list()). +proc_stats() -> + proc_stats(self()). + +-spec(proc_stats(pid()) -> list()). +proc_stats(Pid) -> + Stats = process_info(Pid, [message_queue_len, heap_size, reductions]), + {value, {_, V}, Stats1} = lists:keytake(message_queue_len, 1, Stats), + [{mailbox_len, V} | Stats1]. + +-define(DISABLED, 0). + +init_proc_mng_policy(undefined) -> ok; +init_proc_mng_policy(Zone) -> + #{max_heap_size := MaxHeapSizeInBytes} + = ShutdownPolicy + = emqx_zone:get_env(Zone, force_shutdown_policy), + MaxHeapSize = MaxHeapSizeInBytes div erlang:system_info(wordsize), + _ = erlang:process_flag(max_heap_size, MaxHeapSize), % zero is discarded + erlang:put(force_shutdown_policy, ShutdownPolicy), + ok. + +%% @doc Check self() process status against connection/session process management policy, +%% return `continue | hibernate | {shutdown, Reason}' accordingly. +%% `continue': There is nothing out of the ordinary. +%% `hibernate': Nothing to process in my mailbox, and since this check is triggered +%% by a timer, we assume it is a fat chance to continue idel, hence hibernate. +%% `shutdown': Some numbers (message queue length hit the limit), +%% hence shutdown for greater good (system stability). +-spec(conn_proc_mng_policy(#{message_queue_len => integer()} | false) -> + continue | hibernate | {shutdown, _}). +conn_proc_mng_policy(#{message_queue_len := MaxMsgQueueLen}) -> + Qlength = proc_info(message_queue_len), + Checks = + [{fun() -> is_message_queue_too_long(Qlength, MaxMsgQueueLen) end, + {shutdown, message_queue_too_long}}, + {fun() -> Qlength > 0 end, continue}, + {fun() -> true end, hibernate} + ], + check(Checks); +conn_proc_mng_policy(_) -> + %% disable by default + conn_proc_mng_policy(#{message_queue_len => 0}). + +check([{Pred, Result} | Rest]) -> + case Pred() of + true -> Result; + false -> check(Rest) + end. + +is_message_queue_too_long(Qlength, Max) -> + is_enabled(Max) andalso Qlength > Max. + +is_enabled(Max) -> is_integer(Max) andalso Max > ?DISABLED. + +proc_info(Key) -> + {Key, Value} = erlang:process_info(self(), Key), + Value. + +-spec(drain_down(pos_integer()) -> list(pid())). +drain_down(Cnt) when Cnt > 0 -> + drain_down(Cnt, []). + +drain_down(0, Acc) -> + lists:reverse(Acc); + +drain_down(Cnt, Acc) -> + receive + {'DOWN', _MRef, process, Pid, _Reason} -> + drain_down(Cnt - 1, [Pid|Acc]) + after 0 -> + lists:reverse(Acc) + end. + diff --git a/src/emqx_mod_presence.erl b/src/emqx_mod_presence.erl new file mode 100644 index 000000000..59c675f9a --- /dev/null +++ b/src/emqx_mod_presence.erl @@ -0,0 +1,76 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mod_presence). + +-behaviour(emqx_gen_mod). + +-include("emqx.hrl"). + +-export([load/1, unload/1]). + +-export([on_client_connected/4, on_client_disconnected/3]). + +-define(ATTR_KEYS, [clean_start, proto_ver, proto_name, keepalive]). + +load(Env) -> + emqx_hooks:add('client.connected', fun ?MODULE:on_client_connected/4, [Env]), + emqx_hooks:add('client.disconnected', fun ?MODULE:on_client_disconnected/3, [Env]). + +on_client_connected(#{client_id := ClientId, + username := Username, + peername := {IpAddr, _}}, ConnAck, ConnAttrs, Env) -> + Attrs = lists:filter(fun({K, _}) -> lists:member(K, ?ATTR_KEYS) end, ConnAttrs), + case emqx_json:safe_encode([{clientid, ClientId}, + {username, Username}, + {ipaddress, iolist_to_binary(esockd_net:ntoa(IpAddr))}, + {connack, ConnAck}, + {ts, os:system_time(second)} | Attrs]) of + {ok, Payload} -> + emqx:publish(message(qos(Env), topic(connected, ClientId), Payload)); + {error, Reason} -> + emqx_logger:error("[Presence Module] Json error: ~p", [Reason]) + end. + +on_client_disconnected(#{client_id := ClientId, username := Username}, Reason, Env) -> + case emqx_json:safe_encode([{clientid, ClientId}, + {username, Username}, + {reason, reason(Reason)}, + {ts, os:system_time(second)}]) of + {ok, Payload} -> + emqx_broker:publish(message(qos(Env), topic(disconnected, ClientId), Payload)); + {error, Reason} -> + emqx_logger:error("[Presence Module] Json error: ~p", [Reason]) + end. + +unload(_Env) -> + emqx_hooks:del('client.connected', fun ?MODULE:on_client_connected/4), + emqx_hooks:del('client.disconnected', fun ?MODULE:on_client_disconnected/3). + +message(QoS, Topic, Payload) -> + emqx_message:set_flag( + sys, emqx_message:make( + ?MODULE, QoS, Topic, iolist_to_binary(Payload))). + +topic(connected, ClientId) -> + emqx_topic:systop(iolist_to_binary(["clients/", ClientId, "/connected"])); +topic(disconnected, ClientId) -> + emqx_topic:systop(iolist_to_binary(["clients/", ClientId, "/disconnected"])). + +qos(Env) -> proplists:get_value(qos, Env, 0). + +reason(Reason) when is_atom(Reason) -> Reason; +reason({Error, _}) when is_atom(Error) -> Error; +reason(_) -> internal_error. + diff --git a/src/emqx_mod_rewrite.erl b/src/emqx_mod_rewrite.erl new file mode 100644 index 000000000..25faef166 --- /dev/null +++ b/src/emqx_mod_rewrite.erl @@ -0,0 +1,77 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mod_rewrite). + +-include_lib("emqx.hrl"). +-include_lib("emqx_mqtt.hrl"). + +-export([load/1, unload/1]). + +-export([rewrite_subscribe/3, rewrite_unsubscribe/3, rewrite_publish/2]). + +%%------------------------------------------------------------------------------ +%% Load/Unload +%%------------------------------------------------------------------------------ + +load(RawRules) -> + Rules = compile(RawRules), + emqx_hooks:add('client.subscribe', fun ?MODULE:rewrite_subscribe/3, [Rules]), + emqx_hooks:add('client.unsubscribe', fun ?MODULE:rewrite_unsubscribe/3, [Rules]), + emqx_hooks:add('message.publish', fun ?MODULE:rewrite_publish/2, [Rules]). + +rewrite_subscribe(_Credentials, TopicTable, Rules) -> + {ok, [{match_rule(Topic, Rules), Opts} || {Topic, Opts} <- TopicTable]}. + +rewrite_unsubscribe(_Credentials, TopicTable, Rules) -> + {ok, [{match_rule(Topic, Rules), Opts} || {Topic, Opts} <- TopicTable]}. + +rewrite_publish(Message = #message{topic = Topic}, Rules) -> + {ok, Message#message{topic = match_rule(Topic, Rules)}}. + +unload(_) -> + emqx_hooks:del('client.subscribe', fun ?MODULE:rewrite_subscribe/3), + emqx_hooks:del('client.unsubscribe', fun ?MODULE:rewrite_unsubscribe/3), + emqx_hooks:del('message.publish', fun ?MODULE:rewrite_publish/2). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +match_rule(Topic, []) -> + Topic; + +match_rule(Topic, [{rewrite, Filter, MP, Dest} | Rules]) -> + case emqx_topic:match(Topic, Filter) of + true -> match_regx(Topic, MP, Dest); + false -> match_rule(Topic, Rules) + end. + +match_regx(Topic, MP, Dest) -> + case re:run(Topic, MP, [{capture, all_but_first, list}]) of + {match, Captured} -> + Vars = lists:zip(["\\$" ++ integer_to_list(I) + || I <- lists:seq(1, length(Captured))], Captured), + iolist_to_binary(lists:foldl( + fun({Var, Val}, Acc) -> + re:replace(Acc, Var, Val, [global]) + end, Dest, Vars)); + nomatch -> Topic + end. + +compile(Rules) -> + lists:map(fun({rewrite, Topic, Re, Dest}) -> + {ok, MP} = re:compile(Re), + {rewrite, Topic, MP, Dest} + end, Rules). diff --git a/src/emqx_mod_subscription.erl b/src/emqx_mod_subscription.erl new file mode 100644 index 000000000..aed7f5af2 --- /dev/null +++ b/src/emqx_mod_subscription.erl @@ -0,0 +1,51 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mod_subscription). + +-behaviour(emqx_gen_mod). + +-include_lib("emqx.hrl"). +-include_lib("emqx_mqtt.hrl"). + +-export([load/1, on_session_created/3, unload/1]). + +%%-------------------------------------------------------------------- +%% Load/Unload Hook +%%-------------------------------------------------------------------- + +load(Topics) -> + emqx_hooks:add('session.created', fun ?MODULE:on_session_created/3, [Topics]). + +on_session_created(#{client_id := ClientId}, SessAttrs, Topics) -> + Username = proplists:get_value(username, SessAttrs), + Replace = fun(Topic) -> + rep(<<"%u">>, Username, rep(<<"%c">>, ClientId, Topic)) + end, + emqx_session:subscribe(self(), [{Replace(Topic), #{qos => QoS}} || {Topic, QoS} <- Topics]). + +unload(_) -> + emqx_hooks:del('session.created', fun ?MODULE:on_session_created/3). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +rep(<<"%c">>, ClientId, Topic) -> + emqx_topic:feed_var(<<"%c">>, ClientId, Topic); +rep(<<"%u">>, undefined, Topic) -> + Topic; +rep(<<"%u">>, Username, Topic) -> + emqx_topic:feed_var(<<"%u">>, Username, Topic). + diff --git a/src/emqttd_mod_sup.erl b/src/emqx_mod_sup.erl similarity index 77% rename from src/emqttd_mod_sup.erl rename to src/emqx_mod_sup.erl index b8335e6b3..ca5a3d61c 100644 --- a/src/emqttd_mod_sup.erl +++ b/src/emqx_mod_sup.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,27 +11,17 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_mod_sup). +-module(emqx_mod_sup). -behaviour(supervisor). --include("emqttd.hrl"). - -%% API -export([start_link/0, start_child/1, start_child/2, stop_child/1]). - -%% Supervisor callbacks -export([init/1]). %% Helper macro for declaring children of supervisor -define(CHILD(Mod, Type), {Mod, {Mod, start_link, []}, permanent, 5000, Type, [Mod]}). -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). @@ -49,10 +38,9 @@ stop_child(ChildId) -> Error -> Error end. -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ %% Supervisor callbacks -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ init([]) -> {ok, {{one_for_one, 10, 100}, []}}. - diff --git a/src/emqx_modules.erl b/src/emqx_modules.erl new file mode 100644 index 000000000..60c4b6351 --- /dev/null +++ b/src/emqx_modules.erl @@ -0,0 +1,33 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_modules). + +-export([load/0, unload/0]). + +-spec(load() -> ok). +load() -> + lists:foreach( + fun({Mod, Env}) -> + ok = Mod:load(Env), + logger:info("Load ~s module successfully.", [Mod]) + end, emqx_config:get_env(modules, [])). + +-spec(unload() -> ok). +unload() -> + lists:foreach( + fun({Mod, Env}) -> + Mod:unload(Env) end, + emqx_config:get_env(modules, [])). + diff --git a/src/emqx_mountpoint.erl b/src/emqx_mountpoint.erl new file mode 100644 index 000000000..f2046d5ee --- /dev/null +++ b/src/emqx_mountpoint.erl @@ -0,0 +1,52 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mountpoint). + +-include("emqx.hrl"). + +-export([mount/2, unmount/2]). +-export([replvar/2]). + +-type(mountpoint() :: binary()). +-export_type([mountpoint/0]). + +mount(undefined, Any) -> + Any; +mount(MountPoint, Msg = #message{topic = Topic}) -> + Msg#message{topic = <>}; + +mount(MountPoint, TopicFilters) when is_list(TopicFilters) -> + [{<>, SubOpts} || {Topic, SubOpts} <- TopicFilters]. + +unmount(undefined, Msg) -> + Msg; +unmount(MountPoint, Msg = #message{topic = Topic}) -> + case catch split_binary(Topic, byte_size(MountPoint)) of + {MountPoint, Topic1} -> Msg#message{topic = Topic1}; + _Other -> Msg + end. + +replvar(undefined, _Vars) -> + undefined; +replvar(MountPoint, #{client_id := ClientId, username := Username}) -> + lists:foldl(fun feed_var/2, MountPoint, [{<<"%c">>, ClientId}, {<<"%u">>, Username}]). + +feed_var({<<"%c">>, ClientId}, MountPoint) -> + emqx_topic:feed_var(<<"%c">>, ClientId, MountPoint); +feed_var({<<"%u">>, undefined}, MountPoint) -> + MountPoint; +feed_var({<<"%u">>, Username}, MountPoint) -> + emqx_topic:feed_var(<<"%u">>, Username, MountPoint). + diff --git a/src/emqx_mqtt_caps.erl b/src/emqx_mqtt_caps.erl new file mode 100644 index 000000000..5f919d9c0 --- /dev/null +++ b/src/emqx_mqtt_caps.erl @@ -0,0 +1,147 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc MQTTv5 capabilities +-module(emqx_mqtt_caps). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-export([check_pub/2, check_sub/2]). +-export([get_caps/1, get_caps/2]). + +-type(caps() :: #{max_packet_size => integer(), + max_clientid_len => integer(), + max_topic_alias => integer(), + max_topic_levels => integer(), + max_qos_allowed => emqx_mqtt_types:qos(), + mqtt_retain_available => boolean(), + mqtt_shared_subscription => boolean(), + mqtt_wildcard_subscription => boolean()}). + +-export_type([caps/0]). + +-define(UNLIMITED, 0). +-define(DEFAULT_CAPS, [{max_packet_size, ?MAX_PACKET_SIZE}, + {max_clientid_len, ?MAX_CLIENTID_LEN}, + {max_topic_alias, ?UNLIMITED}, + {max_topic_levels, ?UNLIMITED}, + {max_qos_allowed, ?QOS_2}, + {mqtt_retain_available, true}, + {mqtt_shared_subscription, true}, + {mqtt_wildcard_subscription, true}]). + +-define(PUBCAP_KEYS, [max_qos_allowed, + mqtt_retain_available, + max_topic_alias + ]). +-define(SUBCAP_KEYS, [max_qos_allowed, + max_topic_levels, + mqtt_shared_subscription, + mqtt_wildcard_subscription]). + +-spec(check_pub(emqx_types:zone(), map()) -> ok | {error, emqx_mqtt_types:reason_code()}). +check_pub(Zone, Props) when is_map(Props) -> + do_check_pub(Props, maps:to_list(get_caps(Zone, publish))). + +do_check_pub(_Props, []) -> + ok; +do_check_pub(Props = #{qos := QoS}, [{max_qos_allowed, MaxQoS}|Caps]) -> + case QoS > MaxQoS of + true -> {error, ?RC_QOS_NOT_SUPPORTED}; + false -> do_check_pub(Props, Caps) + end; +do_check_pub(Props = #{ topic_alias := TopicAlias}, [{max_topic_alias, MaxTopicAlias}| Caps]) -> + case TopicAlias =< MaxTopicAlias andalso TopicAlias > 0 of + false -> {error, ?RC_TOPIC_ALIAS_INVALID}; + true -> do_check_pub(Props, Caps) + end; +do_check_pub(#{retain := true}, [{mqtt_retain_available, false}|_Caps]) -> + {error, ?RC_RETAIN_NOT_SUPPORTED}; +do_check_pub(Props, [{max_topic_alias, _} | Caps]) -> + do_check_pub(Props, Caps); +do_check_pub(Props, [{mqtt_retain_available, _}|Caps]) -> + do_check_pub(Props, Caps). + +-spec(check_sub(emqx_types:zone(), emqx_mqtt_types:topic_filters()) + -> {ok | error, emqx_mqtt_types:topic_filters()}). +check_sub(Zone, TopicFilters) -> + Caps = maps:to_list(get_caps(Zone, subscribe)), + lists:foldr(fun({Topic, Opts}, {Ok, Result}) -> + case check_sub(Topic, Opts, Caps) of + {ok, Opts1} -> + {Ok, [{Topic, Opts1}|Result]}; + {error, Opts1} -> + {error, [{Topic, Opts1}|Result]} + end + end, {ok, []}, TopicFilters). + +check_sub(_Topic, Opts, []) -> + {ok, Opts}; +check_sub(Topic, Opts = #{qos := QoS}, [{max_qos_allowed, MaxQoS}|Caps]) -> + check_sub(Topic, Opts#{qos := min(QoS, MaxQoS)}, Caps); +check_sub(Topic, Opts, [{mqtt_shared_subscription, true}|Caps]) -> + check_sub(Topic, Opts, Caps); +check_sub(Topic, Opts, [{mqtt_shared_subscription, false}|Caps]) -> + case maps:is_key(share, Opts) of + true -> + {error, Opts#{rc := ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}}; + false -> check_sub(Topic, Opts, Caps) + end; +check_sub(Topic, Opts, [{mqtt_wildcard_subscription, true}|Caps]) -> + check_sub(Topic, Opts, Caps); +check_sub(Topic, Opts, [{mqtt_wildcard_subscription, false}|Caps]) -> + case emqx_topic:wildcard(Topic) of + true -> + {error, Opts#{rc := ?RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED}}; + false -> check_sub(Topic, Opts, Caps) + end; +check_sub(Topic, Opts, [{max_topic_levels, ?UNLIMITED}|Caps]) -> + check_sub(Topic, Opts, Caps); +check_sub(Topic, Opts, [{max_topic_levels, Limit}|Caps]) -> + case emqx_topic:levels(Topic) of + Levels when Levels > Limit -> + {error, Opts#{rc := ?RC_TOPIC_FILTER_INVALID}}; + _ -> check_sub(Topic, Opts, Caps) + end. + +get_caps(Zone, publish) -> + with_env(Zone, '$mqtt_pub_caps', + fun() -> + filter_caps(?PUBCAP_KEYS, get_caps(Zone)) + end); + +get_caps(Zone, subscribe) -> + with_env(Zone, '$mqtt_sub_caps', + fun() -> + filter_caps(?SUBCAP_KEYS, get_caps(Zone)) + end). + +get_caps(Zone) -> + with_env(Zone, '$mqtt_caps', + fun() -> + maps:from_list([{Cap, emqx_zone:get_env(Zone, Cap, Def)} + || {Cap, Def} <- ?DEFAULT_CAPS]) + end). + +filter_caps(Keys, Caps) -> + maps:filter(fun(Key, _Val) -> lists:member(Key, Keys) end, Caps). + +with_env(Zone, Key, InitFun) -> + case emqx_zone:get_env(Zone, Key) of + undefined -> Caps = InitFun(), + ok = emqx_zone:set_env(Zone, Key, Caps), + Caps; + ZoneCaps -> ZoneCaps + end. diff --git a/src/emqx_mqtt_props.erl b/src/emqx_mqtt_props.erl new file mode 100644 index 000000000..f4374d0d0 --- /dev/null +++ b/src/emqx_mqtt_props.erl @@ -0,0 +1,148 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc MQTT5 Properties +-module(emqx_mqtt_props). + +-include("emqx_mqtt.hrl"). + +-export([id/1, name/1, filter/2, validate/1]). + +-define(PROPS_TABLE, + #{16#01 => {'Payload-Format-Indicator', 'Byte', [?PUBLISH]}, + 16#02 => {'Message-Expiry-Interval', 'Four-Byte-Integer', [?PUBLISH]}, + 16#03 => {'Content-Type', 'UTF8-Encoded-String', [?PUBLISH]}, + 16#08 => {'Response-Topic', 'UTF8-Encoded-String', [?PUBLISH]}, + 16#09 => {'Correlation-Data', 'Binary-Data', [?PUBLISH]}, + 16#0B => {'Subscription-Identifier', 'Variable-Byte-Integer', [?PUBLISH, ?SUBSCRIBE]}, + 16#11 => {'Session-Expiry-Interval', 'Four-Byte-Integer', [?CONNECT, ?CONNACK, ?DISCONNECT]}, + 16#12 => {'Assigned-Client-Identifier', 'UTF8-Encoded-String', [?CONNACK]}, + 16#13 => {'Server-Keep-Alive', 'Two-Byte-Integer', [?CONNACK]}, + 16#15 => {'Authentication-Method', 'UTF8-Encoded-String', [?CONNECT, ?CONNACK, ?AUTH]}, + 16#16 => {'Authentication-Data', 'Binary-Data', [?CONNECT, ?CONNACK, ?AUTH]}, + 16#17 => {'Request-Problem-Information', 'Byte', [?CONNECT]}, + 16#18 => {'Will-Delay-Interval', 'Four-Byte-Integer', ['WILL']}, + 16#19 => {'Request-Response-Information', 'Byte', [?CONNECT]}, + 16#1A => {'Response-Information', 'UTF8-Encoded-String', [?CONNACK]}, + 16#1C => {'Server-Reference', 'UTF8-Encoded-String', [?CONNACK, ?DISCONNECT]}, + 16#1F => {'Reason-String', 'UTF8-Encoded-String', 'ALL'}, + 16#21 => {'Receive-Maximum', 'Two-Byte-Integer', [?CONNECT, ?CONNACK]}, + 16#22 => {'Topic-Alias-Maximum', 'Two-Byte-Integer', [?CONNECT, ?CONNACK]}, + 16#23 => {'Topic-Alias', 'Two-Byte-Integer', [?PUBLISH]}, + 16#24 => {'Maximum-QoS', 'Byte', [?CONNACK]}, + 16#25 => {'Retain-Available', 'Byte', [?CONNACK]}, + 16#26 => {'User-Property', 'UTF8-String-Pair', 'ALL'}, + 16#27 => {'Maximum-Packet-Size', 'Four-Byte-Integer', [?CONNECT, ?CONNACK]}, + 16#28 => {'Wildcard-Subscription-Available', 'Byte', [?CONNACK]}, + 16#29 => {'Subscription-Identifier-Available', 'Byte', [?CONNACK]}, + 16#2A => {'Shared-Subscription-Available', 'Byte', [?CONNACK]}}). + +name(16#01) -> 'Payload-Format-Indicator'; +name(16#02) -> 'Message-Expiry-Interval'; +name(16#03) -> 'Content-Type'; +name(16#08) -> 'Response-Topic'; +name(16#09) -> 'Correlation-Data'; +name(16#0B) -> 'Subscription-Identifier'; +name(16#11) -> 'Session-Expiry-Interval'; +name(16#12) -> 'Assigned-Client-Identifier'; +name(16#13) -> 'Server-Keep-Alive'; +name(16#15) -> 'Authentication-Method'; +name(16#16) -> 'Authentication-Data'; +name(16#17) -> 'Request-Problem-Information'; +name(16#18) -> 'Will-Delay-Interval'; +name(16#19) -> 'Request-Response-Information'; +name(16#1A) -> 'Response-Information'; +name(16#1C) -> 'Server-Reference'; +name(16#1F) -> 'Reason-String'; +name(16#21) -> 'Receive-Maximum'; +name(16#22) -> 'Topic-Alias-Maximum'; +name(16#23) -> 'Topic-Alias'; +name(16#24) -> 'Maximum-QoS'; +name(16#25) -> 'Retain-Available'; +name(16#26) -> 'User-Property'; +name(16#27) -> 'Maximum-Packet-Size'; +name(16#28) -> 'Wildcard-Subscription-Available'; +name(16#29) -> 'Subscription-Identifier-Available'; +name(16#2A) -> 'Shared-Subscription-Available'. + +id('Payload-Format-Indicator') -> 16#01; +id('Message-Expiry-Interval') -> 16#02; +id('Content-Type') -> 16#03; +id('Response-Topic') -> 16#08; +id('Correlation-Data') -> 16#09; +id('Subscription-Identifier') -> 16#0B; +id('Session-Expiry-Interval') -> 16#11; +id('Assigned-Client-Identifier') -> 16#12; +id('Server-Keep-Alive') -> 16#13; +id('Authentication-Method') -> 16#15; +id('Authentication-Data') -> 16#16; +id('Request-Problem-Information') -> 16#17; +id('Will-Delay-Interval') -> 16#18; +id('Request-Response-Information') -> 16#19; +id('Response-Information') -> 16#1A; +id('Server-Reference') -> 16#1C; +id('Reason-String') -> 16#1F; +id('Receive-Maximum') -> 16#21; +id('Topic-Alias-Maximum') -> 16#22; +id('Topic-Alias') -> 16#23; +id('Maximum-QoS') -> 16#24; +id('Retain-Available') -> 16#25; +id('User-Property') -> 16#26; +id('Maximum-Packet-Size') -> 16#27; +id('Wildcard-Subscription-Available') -> 16#28; +id('Subscription-Identifier-Available') -> 16#29; +id('Shared-Subscription-Available') -> 16#2A. + +filter(PacketType, Props) when is_map(Props) -> + maps:from_list(filter(PacketType, maps:to_list(Props))); + +filter(PacketType, Props) when ?CONNECT =< PacketType, PacketType =< ?AUTH, is_list(Props) -> + Filter = fun(Name) -> + case maps:find(id(Name), ?PROPS_TABLE) of + {ok, {Name, _Type, 'ALL'}} -> + true; + {ok, {Name, _Type, AllowedTypes}} -> + lists:member(PacketType, AllowedTypes); + error -> false + end + end, + [Prop || Prop = {Name, _} <- Props, Filter(Name)]. + +validate(Props) when is_map(Props) -> + lists:foreach(fun validate_prop/1, maps:to_list(Props)). + +validate_prop(Prop = {Name, Val}) -> + case maps:find(id(Name), ?PROPS_TABLE) of + {ok, {Name, Type, _}} -> + validate_value(Type, Val) + orelse error(bad_property, Prop); + error -> + error({bad_property, Prop}) + end. + +validate_value('Byte', Val) -> + is_integer(Val); +validate_value('Two-Byte-Integer', Val) -> + is_integer(Val); +validate_value('Four-Byte-Integer', Val) -> + is_integer(Val); +validate_value('Variable-Byte-Integer', Val) -> + is_integer(Val); +validate_value('UTF8-Encoded-String', Val) -> + is_binary(Val); +validate_value('Binary-Data', Val) -> + is_binary(Val); +validate_value('UTF8-String-Pair', Val) -> + is_tuple(Val) orelse is_list(Val). + diff --git a/src/emqx_mqtt_types.erl b/src/emqx_mqtt_types.erl new file mode 100644 index 000000000..71451cca7 --- /dev/null +++ b/src/emqx_mqtt_types.erl @@ -0,0 +1,42 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mqtt_types). + +-include("emqx_mqtt.hrl"). + +-export_type([version/0, qos/0, qos_name/0]). +-export_type([connack/0, reason_code/0]). +-export_type([properties/0, subopts/0]). +-export_type([topic_filters/0]). +-export_type([packet_id/0, packet_type/0, packet/0]). + +-type(qos() :: ?QOS_0 | ?QOS_1 | ?QOS_2). +-type(version() :: ?MQTT_PROTO_V3 | ?MQTT_PROTO_V4 | ?MQTT_PROTO_V5). +-type(qos_name() :: qos0 | at_most_once | + qos1 | at_least_once | + qos2 | exactly_once). +-type(packet_type() :: ?RESERVED..?AUTH). +-type(connack() :: ?CONNACK_ACCEPT..?CONNACK_AUTH). +-type(reason_code() :: 0..16#FF). +-type(packet_id() :: 1..16#FFFF). +-type(properties() :: #{atom() => term()}). +-type(subopts() :: #{rh := 0 | 1 | 2, + rap := 0 | 1, + nl := 0 | 1, + qos := qos(), + rc => reason_code() + }). +-type(topic_filters() :: [{emqx_topic:topic(), subopts()}]). +-type(packet() :: #mqtt_packet{}). diff --git a/src/emqx_mqueue.erl b/src/emqx_mqueue.erl new file mode 100644 index 000000000..48b0fa439 --- /dev/null +++ b/src/emqx_mqueue.erl @@ -0,0 +1,168 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc A Simple in-memory message queue. +%% +%% Notice that MQTT is not a (on-disk) persistent messaging queue. +%% It assumes that clients should be online in most of the time. +%% +%% This module implements a simple in-memory queue for MQTT persistent session. +%% +%% If the broker restarts or crashes, all queued messages will be lost. +%% +%% Concept of Message Queue and Inflight Window: +%% +%% |<----------------- Max Len ----------------->| +%% ----------------------------------------------- +%% IN -> | Messages Queue | Inflight Window | -> Out +%% ----------------------------------------------- +%% |<--- Win Size --->| +%% +%% +%% 1. Inflight Window is to store the messages +%% that are delivered but still awaiting for puback. +%% +%% 2. Messages are enqueued to tail when the inflight window is full. +%% +%% 3. QoS=0 messages are only enqueued when `store_qos0' is given `true` +%% in init options +%% +%% 4. If the queue is full, drop the oldest one +%% unless `max_len' is set to `0' which implies (`infinity'). +%% +%% @end + +-module(emqx_mqueue). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-export([init/1]). +-export([is_empty/1]). +-export([len/1, max_len/1]). +-export([in/2, out/1]). +-export([stats/1, dropped/1]). + +-export_type([mqueue/0, options/0]). + +-type(topic() :: emqx_topic:topic()). +-type(priority() :: infinity | integer()). +-type(pq() :: emqx_pqueue:q()). +-type(count() :: non_neg_integer()). +-type(p_table() :: ?NO_PRIORITY_TABLE | #{topic() := priority()}). +-type(options() :: #{max_len := count(), + priorities => p_table(), + default_priority => highest | lowest, + store_qos0 => boolean() + }). +-type(message() :: pemqx_types:message()). + +-type(stat() :: {len, non_neg_integer()} + | {max_len, non_neg_integer()} + | {dropped, non_neg_integer()}). + +-define(PQUEUE, emqx_pqueue). +-define(LOWEST_PRIORITY, 0). +-define(HIGHEST_PRIORITY, infinity). +-define(MAX_LEN_INFINITY, 0). + +-record(mqueue, { + store_qos0 = false :: boolean(), + max_len = ?MAX_LEN_INFINITY :: count(), + len = 0 :: count(), + dropped = 0 :: count(), + p_table = ?NO_PRIORITY_TABLE :: p_table(), + default_p = ?LOWEST_PRIORITY :: priority(), + q = ?PQUEUE:new() :: pq() + }). + +-opaque(mqueue() :: #mqueue{}). + +-spec(init(options()) -> mqueue()). +init(Opts = #{max_len := MaxLen0, store_qos0 := QoS_0}) -> + MaxLen = case (is_integer(MaxLen0) andalso MaxLen0 > ?MAX_LEN_INFINITY) of + true -> MaxLen0; + false -> ?MAX_LEN_INFINITY + end, + #mqueue{max_len = MaxLen, + store_qos0 = QoS_0, + p_table = get_opt(priorities, Opts, ?NO_PRIORITY_TABLE), + default_p = get_priority_opt(Opts) + }. + +is_empty(#mqueue{len = Len}) -> Len =:= 0. + +len(#mqueue{len = Len}) -> Len. + +max_len(#mqueue{max_len = MaxLen}) -> MaxLen. + +%% @doc Return number of dropped messages. +-spec(dropped(mqueue()) -> count()). +dropped(#mqueue{dropped = Dropped}) -> Dropped. + +%% @doc Stats of the mqueue +-spec(stats(mqueue()) -> [stat()]). +stats(#mqueue{max_len = MaxLen, dropped = Dropped} = MQ) -> + [{len, len(MQ)}, {max_len, MaxLen}, {dropped, Dropped}]. + +%% @doc Enqueue a message. +-spec(in(message(), mqueue()) -> {undefined | message(), mqueue()}). +in(#message{qos = ?QOS_0}, MQ = #mqueue{store_qos0 = false}) -> + {_Dropped = undefined, MQ}; +in(Msg = #message{topic = Topic}, MQ = #mqueue{default_p = Dp, + p_table = PTab, + q = Q, + len = Len, + max_len = MaxLen, + dropped = Dropped + } = MQ) -> + Priority = get_priority(Topic, PTab, Dp), + PLen = ?PQUEUE:plen(Priority, Q), + case MaxLen =/= ?MAX_LEN_INFINITY andalso PLen =:= MaxLen of + true -> + %% reached max length, drop the oldest message + {{value, DroppedMsg}, Q1} = ?PQUEUE:out(Priority, Q), + Q2 = ?PQUEUE:in(Msg, Priority, Q1), + {DroppedMsg, MQ#mqueue{q = Q2, dropped = Dropped + 1}}; + false -> + {_DroppedMsg = undefined, MQ#mqueue{len = Len + 1, q = ?PQUEUE:in(Msg, Priority, Q)}} + end. + +-spec(out(mqueue()) -> {empty | {value, message()}, mqueue()}). +out(MQ = #mqueue{len = 0, q = Q}) -> + 0 = ?PQUEUE:len(Q), %% assert, in this case, ?PQUEUE:len should be very cheap + {empty, MQ}; +out(MQ = #mqueue{q = Q, len = Len}) -> + {R, Q1} = ?PQUEUE:out(Q), + {R, MQ#mqueue{q = Q1, len = Len - 1}}. + +get_opt(Key, Opts, Default) -> + case maps:get(Key, Opts, Default) of + undefined -> Default; + X -> X + end. + +get_priority_opt(Opts) -> + case get_opt(default_priority, Opts, ?LOWEST_PRIORITY) of + lowest -> ?LOWEST_PRIORITY; + highest -> ?HIGHEST_PRIORITY; + N when is_integer(N) -> N + end. + +%% MICRO-OPTIMIZATION: When there is no priority table defined (from config), +%% disregard default priority from config, always use lowest (?LOWEST_PRIORITY=0) +%% because the lowest priority in emqx_pqueue is a fallback to queue:queue() +%% while the highest 'infinity' is a [{infinity, queue:queue()}] +get_priority(_Topic, ?NO_PRIORITY_TABLE, _) -> ?LOWEST_PRIORITY; +get_priority(Topic, PTab, Dp) -> maps:get(Topic, PTab, Dp). diff --git a/src/emqx_packet.erl b/src/emqx_packet.erl new file mode 100644 index 000000000..e9c2fa0df --- /dev/null +++ b/src/emqx_packet.erl @@ -0,0 +1,256 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_packet). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-export([protocol_name/1]). +-export([type_name/1]). +-export([validate/1]). +-export([format/1]). +-export([to_message/2, from_message/2]). +-export([will_msg/1]). + +%% @doc Protocol name of version +-spec(protocol_name(emqx_mqtt_types:version()) -> binary()). +protocol_name(?MQTT_PROTO_V3) -> + <<"MQIsdp">>; +protocol_name(?MQTT_PROTO_V4) -> + <<"MQTT">>; +protocol_name(?MQTT_PROTO_V5) -> + <<"MQTT">>. + +%% @doc Name of MQTT packet type +-spec(type_name(emqx_mqtt_types:packet_type()) -> atom()). +type_name(Type) when Type > ?RESERVED andalso Type =< ?AUTH -> + lists:nth(Type, ?TYPE_NAMES). + +%%------------------------------------------------------------------------------ +%% Validate MQTT Packet +%%------------------------------------------------------------------------------ + +validate(?SUBSCRIBE_PACKET(_PacketId, _Properties, [])) -> + error(topic_filters_invalid); +validate(?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters)) -> + validate_packet_id(PacketId) + andalso validate_properties(?SUBSCRIBE, Properties) + andalso ok == lists:foreach(fun validate_subscription/1, TopicFilters); + +validate(?UNSUBSCRIBE_PACKET(_PacketId, [])) -> + error(topic_filters_invalid); +validate(?UNSUBSCRIBE_PACKET(PacketId, TopicFilters)) -> + validate_packet_id(PacketId) + andalso ok == lists:foreach(fun emqx_topic:validate/1, TopicFilters); + +validate(?PUBLISH_PACKET(_QoS, <<>>, _, #{'Topic-Alias':= _I}, _)) -> + true; +validate(?PUBLISH_PACKET(_QoS, <<>>, _, _, _)) -> + error(topic_name_invalid); +validate(?PUBLISH_PACKET(_QoS, Topic, _, Properties, _)) -> + ((not emqx_topic:wildcard(Topic)) orelse error(topic_name_invalid)) + andalso validate_properties(?PUBLISH, Properties); + +validate(?CONNECT_PACKET(#mqtt_packet_connect{properties = Properties})) -> + validate_properties(?CONNECT, Properties); + +validate(_Packet) -> + true. + +validate_packet_id(0) -> + error(packet_id_invalid); +validate_packet_id(_) -> + true. + +validate_properties(?SUBSCRIBE, #{'Subscription-Identifier' := I}) + when I =< 0; I >= 16#FFFFFFF -> + error(subscription_identifier_invalid); +validate_properties(?PUBLISH, #{'Topic-Alias':= I}) + when I =:= 0 -> + error(topic_alias_invalid); +validate_properties(?PUBLISH, #{'Subscription-Identifier' := _I}) -> + error(protocol_error); +validate_properties(?PUBLISH, #{'Response-Topic' := ResponseTopic}) -> + case emqx_topic:wildcard(ResponseTopic) of + true -> + error(protocol_error); + false -> + true + end; +validate_properties(?CONNECT, #{'Receive-Maximum' := 0}) -> + error(protocol_error); +validate_properties(?CONNECT, #{'Request-Response-Information' := ReqRespInfo}) + when ReqRespInfo =/= 0, ReqRespInfo =/= 1 -> + error(protocol_error); +validate_properties(?CONNECT, #{'Request-Problem-Information' := ReqProInfo}) + when ReqProInfo =/= 0, ReqProInfo =/= 1 -> + error(protocol_error); +validate_properties(_, _) -> + true. + +validate_subscription({Topic, #{qos := QoS}}) -> + emqx_topic:validate(filter, Topic) andalso validate_qos(QoS). + +validate_qos(QoS) when ?QOS_0 =< QoS, QoS =< ?QOS_2 -> + true; +validate_qos(_) -> error(bad_qos). + +%% @doc From message to packet +-spec(from_message(emqx_mqtt_types:packet_id(), emqx_types:message()) -> emqx_mqtt_types:packet()). +from_message(PacketId, #message{qos = QoS, flags = Flags, headers = Headers, + topic = Topic, payload = Payload}) -> + Flags1 = if Flags =:= undefined -> + #{}; + true -> Flags + end, + Dup = maps:get(dup, Flags1, false), + Retain = maps:get(retain, Flags1, false), + Publish = #mqtt_packet_publish{topic_name = Topic, + packet_id = PacketId, + properties = publish_props(Headers)}, + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + dup = Dup, + qos = QoS, + retain = Retain}, + variable = Publish, payload = Payload}. + +publish_props(Headers) -> + maps:with(['Payload-Format-Indicator', + 'Response-Topic', + 'Correlation-Data', + 'User-Property', + 'Subscription-Identifier', + 'Content-Type', + 'Message-Expiry-Interval'], Headers). + +%% @doc Message from Packet +-spec(to_message(emqx_types:credentials(), emqx_mqtt_types:packet()) + -> emqx_types:message()). +to_message(#{client_id := ClientId, username := Username}, + #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + retain = Retain, + qos = QoS, + dup = Dup}, + variable = #mqtt_packet_publish{topic_name = Topic, + properties = Props}, + payload = Payload}) -> + Msg = emqx_message:make(ClientId, QoS, Topic, Payload), + Msg#message{flags = #{dup => Dup, retain => Retain}, + headers = merge_props(#{username => Username}, Props)}. + +-spec(will_msg(#mqtt_packet_connect{}) -> emqx_types:message()). +will_msg(#mqtt_packet_connect{will_flag = false}) -> + undefined; +will_msg(#mqtt_packet_connect{client_id = ClientId, + username = Username, + will_retain = Retain, + will_qos = QoS, + will_topic = Topic, + will_props = Properties, + will_payload = Payload}) -> + Msg = emqx_message:make(ClientId, QoS, Topic, Payload), + Msg#message{flags = #{dup => false, retain => Retain}, + headers = merge_props(#{username => Username}, Properties)}. + +merge_props(Headers, undefined) -> + Headers; +merge_props(Headers, Props) -> + maps:merge(Headers, Props). + +%% @doc Format packet +-spec(format(emqx_mqtt_types:packet()) -> iolist()). +format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}) -> + format_header(Header, format_variable(Variable, Payload)). + +format_header(#mqtt_packet_header{type = Type, + dup = Dup, + qos = QoS, + retain = Retain}, S) -> + S1 = if + S == undefined -> <<>>; + true -> [", ", S] + end, + io_lib:format("~s(Q~p, R~p, D~p~s)", [type_name(Type), QoS, i(Retain), i(Dup), S1]). + +format_variable(undefined, _) -> + undefined; +format_variable(Variable, undefined) -> + format_variable(Variable); +format_variable(Variable, Payload) -> + io_lib:format("~s, Payload=~p", [format_variable(Variable), Payload]). + +format_variable(#mqtt_packet_connect{ + proto_ver = ProtoVer, + proto_name = ProtoName, + will_retain = WillRetain, + will_qos = WillQoS, + will_flag = WillFlag, + clean_start = CleanStart, + keepalive = KeepAlive, + client_id = ClientId, + will_topic = WillTopic, + will_payload = WillPayload, + username = Username, + password = Password}) -> + Format = "ClientId=~s, ProtoName=~s, ProtoVsn=~p, CleanStart=~s, KeepAlive=~p, Username=~s, Password=~s", + Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)], + {Format1, Args1} = if + WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~s, Payload=~p)", + Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]}; + true -> {Format, Args} + end, + io_lib:format(Format1, Args1); + +format_variable(#mqtt_packet_disconnect + {reason_code = ReasonCode}) -> + io_lib:format("ReasonCode=~p", [ReasonCode]); + +format_variable(#mqtt_packet_connack{ack_flags = AckFlags, + reason_code = ReasonCode}) -> + io_lib:format("AckFlags=~p, ReasonCode=~p", [AckFlags, ReasonCode]); + +format_variable(#mqtt_packet_publish{topic_name = TopicName, + packet_id = PacketId}) -> + io_lib:format("Topic=~s, PacketId=~p", [TopicName, PacketId]); + +format_variable(#mqtt_packet_puback{packet_id = PacketId}) -> + io_lib:format("PacketId=~p", [PacketId]); + +format_variable(#mqtt_packet_subscribe{packet_id = PacketId, + topic_filters = TopicFilters}) -> + io_lib:format("PacketId=~p, TopicFilters=~p", [PacketId, TopicFilters]); + +format_variable(#mqtt_packet_unsubscribe{packet_id = PacketId, + topic_filters = Topics}) -> + io_lib:format("PacketId=~p, TopicFilters=~p", [PacketId, Topics]); + +format_variable(#mqtt_packet_suback{packet_id = PacketId, + reason_codes = ReasonCodes}) -> + io_lib:format("PacketId=~p, ReasonCodes=~p", [PacketId, ReasonCodes]); + +format_variable(#mqtt_packet_unsuback{packet_id = PacketId}) -> + io_lib:format("PacketId=~p", [PacketId]); + +format_variable(PacketId) when is_integer(PacketId) -> + io_lib:format("PacketId=~p", [PacketId]); + +format_variable(undefined) -> undefined. + +format_password(undefined) -> undefined; +format_password(_Password) -> '******'. + +i(true) -> 1; +i(false) -> 0; +i(I) when is_integer(I) -> I. diff --git a/src/emqx_pd.erl b/src/emqx_pd.erl new file mode 100644 index 000000000..ce1e7723c --- /dev/null +++ b/src/emqx_pd.erl @@ -0,0 +1,33 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc The utility functions for erlang process dictionary. +-module(emqx_pd). + +-export([update_counter/2, get_counter/1, reset_counter/1]). + +-type(key() :: term()). + +-spec(update_counter(key(), number()) -> undefined | number()). +update_counter(Key, Inc) -> + put(Key, get_counter(Key) + Inc). + +-spec(get_counter(key()) -> number()). +get_counter(Key) -> + case get(Key) of undefined -> 0; Cnt -> Cnt end. + +-spec(reset_counter(key()) -> number()). +reset_counter(Key) -> + case put(Key, 0) of undefined -> 0; Cnt -> Cnt end. + diff --git a/src/emqttd_plugins.erl b/src/emqx_plugins.erl similarity index 54% rename from src/emqttd_plugins.erl rename to src/emqx_plugins.erl index 50bbce0e5..120ed835c 100644 --- a/src/emqttd_plugins.erl +++ b/src/emqx_plugins.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,13 +11,10 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_plugins). +-module(emqx_plugins). --author("Feng Lee "). - --include("emqttd.hrl"). +-include("emqx.hrl"). -export([init/0]). @@ -28,16 +24,17 @@ -export([list/0]). +-export([load_expand_plugin/1]). + %% @doc Init plugins' config -spec(init() -> ok). init() -> - case emqttd:env(plugins_etc_dir) of - {ok, PluginsEtc} -> + case emqx_config:get_env(plugins_etc_dir) of + undefined -> ok; + PluginsEtc -> CfgFiles = [filename:join(PluginsEtc, File) || - File <- filelib:wildcard("*.config", PluginsEtc)], - lists:foreach(fun init_config/1, CfgFiles); - undefined -> - ok + File <- filelib:wildcard("*.config", PluginsEtc)], + lists:foreach(fun init_config/1, CfgFiles) end. init_config(CfgFile) -> @@ -49,13 +46,73 @@ init_config(CfgFile) -> %% @doc Load all plugins when the broker started. -spec(load() -> list() | {error, term()}). load() -> - case emqttd:env(plugins_loaded_file) of - {ok, File} -> + load_expand_plugins(), + case emqx_config:get_env(plugins_loaded_file) of + undefined -> %% No plugins available + ignore; + File -> ensure_file(File), - with_loaded_file(File, fun(Names) -> load_plugins(Names, false) end); - undefined -> - %% No plugins available - ignore + with_loaded_file(File, fun(Names) -> load_plugins(Names, false) end) + end. + +load_expand_plugins() -> + case emqx_config:get_env(expand_plugins_dir) of + undefined -> ok; + Dir -> + PluginsDir = filelib:wildcard("*", Dir), + lists:foreach(fun(PluginDir) -> + case filelib:is_dir(Dir ++ PluginDir) of + true -> load_expand_plugin(Dir ++ PluginDir); + false -> ok + end + end, PluginsDir) + end. + +load_expand_plugin(PluginDir) -> + init_expand_plugin_config(PluginDir), + Ebin = PluginDir ++ "/ebin", + code:add_patha(Ebin), + Modules = filelib:wildcard(Ebin ++ "/*.beam"), + lists:foreach(fun(Mod) -> + Module = list_to_atom(filename:basename(Mod, ".beam")), + code:load_file(Module) + end, Modules), + case filelib:wildcard(Ebin ++ "/*.app") of + [App|_] -> application:load(list_to_atom(filename:basename(App, ".app"))); + _ -> emqx_logger:error("App file cannot be found."), + {error, load_app_fail} + end. + +init_expand_plugin_config(PluginDir) -> + Priv = PluginDir ++ "/priv", + Etc = PluginDir ++ "/etc", + Schema = filelib:wildcard(Priv ++ "/*.schema"), + Conf = case filelib:wildcard(Etc ++ "/*.conf") of + [] -> []; + [Conf1] -> cuttlefish_conf:file(Conf1) + end, + AppsEnv = cuttlefish_generator:map(cuttlefish_schema:files(Schema), Conf), + lists:foreach(fun({AppName, Envs}) -> + [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs] + end, AppsEnv). + +get_expand_plugin_config() -> + case emqx_config:get_env(expand_plugins_dir) of + undefined -> ok; + Dir -> + PluginsDir = filelib:wildcard("*", Dir), + lists:foldl(fun(PluginDir, Acc) -> + case filelib:is_dir(Dir ++ PluginDir) of + true -> + Etc = Dir ++ PluginDir ++ "/etc", + case filelib:wildcard("*.{conf,config}", Etc) of + [] -> Acc; + [Conf] -> [Conf | Acc] + end; + false -> + Acc + end + end, [], PluginsDir) end. ensure_file(File) -> @@ -66,7 +123,7 @@ with_loaded_file(File, SuccFun) -> {ok, Names} -> SuccFun(Names); {error, Error} -> - lager:error("Failed to read: ~p, error: ~p", [File, Error]), + emqx_logger:error("[Plugins] Failed to read: ~p, error: ~p", [File, Error]), {error, Error} end. @@ -74,7 +131,7 @@ load_plugins(Names, Persistent) -> Plugins = list(), NotFound = Names -- names(Plugins), case NotFound of [] -> ok; - NotFound -> lager:error("Cannot find plugins: ~p", [NotFound]) + NotFound -> emqx_logger:error("[Plugins] Cannot find plugins: ~p", [NotFound]) end, NeedToLoad = Names -- NotFound -- names(started_app), [load_plugin(find_plugin(Name, Plugins), Persistent) || Name <- NeedToLoad]. @@ -82,33 +139,33 @@ load_plugins(Names, Persistent) -> %% @doc Unload all plugins before broker stopped. -spec(unload() -> list() | {error, term()}). unload() -> - case emqttd:env(plugins_loaded_file) of - {ok, File} -> - with_loaded_file(File, fun stop_plugins/1); + case emqx_config:get_env(plugins_loaded_file) of undefined -> - ignore + ignore; + File -> + with_loaded_file(File, fun stop_plugins/1) end. -%% stop plugins +%% Stop plugins stop_plugins(Names) -> [stop_app(App) || App <- Names]. %% @doc List all available plugins --spec(list() -> [mqtt_plugin()]). +-spec(list() -> [emqx_types:plugin()]). list() -> - case emqttd:env(plugins_etc_dir) of - {ok, PluginsEtc} -> - CfgFiles = filelib:wildcard("*.{conf,config}", PluginsEtc), - Plugins = all_plugin_attrs(CfgFiles), + case emqx_config:get_env(plugins_etc_dir) of + undefined -> + []; + PluginsEtc -> + CfgFiles = filelib:wildcard("*.{conf,config}", PluginsEtc) ++ get_expand_plugin_config(), + Plugins = [plugin(CfgFile) || CfgFile <- CfgFiles], StartedApps = names(started_app), - lists:map(fun(Plugin = #mqtt_plugin{name = Name}) -> + lists:map(fun(Plugin = #plugin{name = Name}) -> case lists:member(Name, StartedApps) of - true -> Plugin#mqtt_plugin{active = true}; + true -> Plugin#plugin{active = true}; false -> Plugin end - end, Plugins); - undefined -> - [] + end, Plugins) end. all_plugin_attrs(CfgFiles) -> @@ -122,32 +179,29 @@ all_plugin_attrs(CfgFiles) -> plugin(CfgFile) -> AppName = app_name(CfgFile), - case application:get_all_key(AppName) of - {ok, Attrs} -> - Ver = proplists:get_value(vsn, Attrs, "0"), - Descr = proplists:get_value(description, Attrs, ""), - #mqtt_plugin{name = AppName, version = Ver, descr = Descr}; - _ -> not_found - end. + {ok, Attrs} = application:get_all_key(AppName), + Ver = proplists:get_value(vsn, Attrs, "0"), + Descr = proplists:get_value(description, Attrs, ""), + #plugin{name = AppName, version = Ver, descr = Descr}. %% @doc Load a Plugin -spec(load(atom()) -> ok | {error, term()}). load(PluginName) when is_atom(PluginName) -> case lists:member(PluginName, names(started_app)) of true -> - lager:error("Plugin ~p is already started", [PluginName]), + emqx_logger:error("[Plugins] Plugin ~s is already started", [PluginName]), {error, already_started}; false -> case find_plugin(PluginName) of false -> - lager:error("Plugin ~s not found", [PluginName]), + emqx_logger:error("[Plugins] Plugin ~s not found", [PluginName]), {error, not_found}; Plugin -> load_plugin(Plugin, true) end end. -load_plugin(#mqtt_plugin{name = Name}, Persistent) -> +load_plugin(#plugin{name = Name}, Persistent) -> case load_app(Name) of ok -> start_app(Name, fun(App) -> plugin_loaded(App, Persistent) end); @@ -168,12 +222,12 @@ load_app(App) -> start_app(App, SuccFun) -> case application:ensure_all_started(App) of {ok, Started} -> - lager:info("Started Apps: ~p", [Started]), - lager:info("Load plugin ~p successfully", [App]), + emqx_logger:info("Started Apps: ~p", [Started]), + emqx_logger:info("Load plugin ~s successfully", [App]), SuccFun(App), {ok, Started}; {error, {ErrApp, Reason}} -> - lager:error("load plugin ~p error, cannot start app ~s for ~p", [App, ErrApp, Reason]), + emqx_logger:error("Load plugin ~s error, cannot start app ~s for ~p", [App, ErrApp, Reason]), {error, {ErrApp, Reason}} end. @@ -190,10 +244,10 @@ unload(PluginName) when is_atom(PluginName) -> {true, true} -> unload_plugin(PluginName, true); {false, _} -> - lager:error("Plugin ~p is not started", [PluginName]), + emqx_logger:error("Plugin ~s is not started", [PluginName]), {error, not_started}; {true, false} -> - lager:error("~s is not a plugin, cannot unload it", [PluginName]), + emqx_logger:error("~s is not a plugin, cannot unload it", [PluginName]), {error, not_found} end. @@ -208,11 +262,11 @@ unload_plugin(App, Persistent) -> stop_app(App) -> case application:stop(App) of ok -> - lager:info("Stop plugin ~p successfully~n", [App]), ok; + emqx_logger:info("Stop plugin ~s successfully", [App]), ok; {error, {not_started, App}} -> - lager:error("Plugin ~p is not started~n", [App]), ok; + emqx_logger:error("Plugin ~s is not started", [App]), ok; {error, Reason} -> - lager:error("Stop plugin ~p error: ~p", [App]), {error, Reason} + emqx_logger:error("Stop plugin ~s error: ~p", [App]), {error, Reason} end. %%-------------------------------------------------------------------- @@ -229,7 +283,7 @@ names(started_app) -> [Name || {Name, _Descr, _Ver} <- application:which_applications()]; names(Plugins) -> - [Name || #mqtt_plugin{name = Name} <- Plugins]. + [Name || #plugin{name = Name} <- Plugins]. plugin_loaded(_Name, false) -> ok; @@ -244,7 +298,7 @@ plugin_loaded(Name, true) -> ignore end; {error, Error} -> - lager:error("Cannot read loaded plugins: ~p", [Error]) + emqx_logger:error("Cannot read loaded plugins: ~p", [Error]) end. plugin_unloaded(_Name, false) -> @@ -256,28 +310,29 @@ plugin_unloaded(Name, true) -> true -> write_loaded(lists:delete(Name, Names)); false -> - lager:error("Cannot find ~s in loaded_file", [Name]) + emqx_logger:error("Cannot find ~s in loaded_file", [Name]) end; {error, Error} -> - lager:error("Cannot read loaded_plugins: ~p", [Error]) + emqx_logger:error("Cannot read loaded_plugins: ~p", [Error]) end. read_loaded() -> - case emqttd:env(plugins_loaded_file) of - {ok, File} -> read_loaded(File); - undefined -> {error, not_found} + case emqx_config:get_env(plugins_loaded_file) of + undefined -> {error, not_found}; + File -> read_loaded(File) end. read_loaded(File) -> file:consult(File). write_loaded(AppNames) -> - {ok, File} = emqttd:env(plugins_loaded_file), + File = emqx_config:get_env(plugins_loaded_file), case file:open(File, [binary, write]) of {ok, Fd} -> lists:foreach(fun(Name) -> file:write(Fd, iolist_to_binary(io_lib:format("~s.~n", [Name]))) end, AppNames); {error, Error} -> - lager:error("Open File ~p Error: ~p", [File, Error]), + emqx_logger:error("Open File ~p Error: ~p", [File, Error]), {error, Error} end. + diff --git a/src/emqx_pmon.erl b/src/emqx_pmon.erl new file mode 100644 index 000000000..a00212ed5 --- /dev/null +++ b/src/emqx_pmon.erl @@ -0,0 +1,81 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pmon). + +-compile({no_auto_import, [monitor/3]}). + +-export([new/0]). +-export([monitor/2, monitor/3]). +-export([demonitor/2]). +-export([find/2]). +-export([erase/2, erase_all/2]). +-export([count/1]). + +-type(pmon() :: {?MODULE, map()}). +-export_type([pmon/0]). + +-spec(new() -> pmon()). +new() -> + {?MODULE, maps:new()}. + +-spec(monitor(pid(), pmon()) -> pmon()). +monitor(Pid, PM) -> + ?MODULE:monitor(Pid, undefined, PM). + +-spec(monitor(pid(), term(), pmon()) -> pmon()). +monitor(Pid, Val, {?MODULE, PM}) -> + {?MODULE, case maps:is_key(Pid, PM) of + true -> PM; + false -> Ref = erlang:monitor(process, Pid), + maps:put(Pid, {Ref, Val}, PM) + end}. + +-spec(demonitor(pid(), pmon()) -> pmon()). +demonitor(Pid, {?MODULE, PM}) -> + {?MODULE, case maps:find(Pid, PM) of + {ok, {Ref, _Val}} -> + %% flush + _ = erlang:demonitor(Ref, [flush]), + maps:remove(Pid, PM); + error -> PM + end}. + +-spec(find(pid(), pmon()) -> error | {ok, term()}). +find(Pid, {?MODULE, PM}) -> + case maps:find(Pid, PM) of + {ok, {_Ref, Val}} -> + {ok, Val}; + error -> error + end. + +-spec(erase(pid(), pmon()) -> pmon()). +erase(Pid, {?MODULE, PM}) -> + {?MODULE, maps:remove(Pid, PM)}. + +-spec(erase_all([pid()], pmon()) -> {[{pid(), term()}], pmon()}). +erase_all(Pids, PMon0) -> + lists:foldl( + fun(Pid, {Acc, PMon}) -> + case find(Pid, PMon) of + {ok, Val} -> + {[{Pid, Val}|Acc], erase(Pid, PMon)}; + error -> {Acc, PMon} + end + end, {[], PMon0}, Pids). + +-spec(count(pmon()) -> non_neg_integer()). +count({?MODULE, PM}) -> + maps:size(PM). + diff --git a/src/emqx_pool.erl b/src/emqx_pool.erl new file mode 100644 index 000000000..7b12bea69 --- /dev/null +++ b/src/emqx_pool.erl @@ -0,0 +1,117 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pool). + +-behaviour(gen_server). + +-export([start_link/0, start_link/2]). +-export([submit/1, submit/2]). +-export([async_submit/1, async_submit/2]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(POOL, ?MODULE). + +-type(task() :: fun() | mfa() | {fun(), Args :: list(any())}). + +%% @doc Start pooler supervisor. +start_link() -> + emqx_pool_sup:start_link(?POOL, random, {?MODULE, start_link, []}). + +%% @doc Start pool. +-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). +start_link(Pool, Id) -> + gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, + ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). + +%% @doc Submit work to the pool. +-spec(submit(task()) -> any()). +submit(Task) -> + call({submit, Task}). + +-spec(submit(fun(), list(any())) -> any()). +submit(Fun, Args) -> + call({submit, {Fun, Args}}). + +%% @private +call(Req) -> + gen_server:call(worker(), Req, infinity). + +%% @doc Submit work to the pool asynchronously. +-spec(async_submit(task()) -> ok). +async_submit(Task) -> + cast({async_submit, Task}). + +-spec(async_submit(fun(), list(any())) -> ok). +async_submit(Fun, Args) -> + cast({async_submit, {Fun, Args}}). + +%% @private +cast(Msg) -> + gen_server:cast(worker(), Msg). + +%% @private +worker() -> + gproc_pool:pick_worker(?POOL). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Pool, Id]) -> + true = gproc_pool:connect_worker(Pool, {Pool, Id}), + {ok, #{pool => Pool, id => Id}}. + +handle_call({submit, Task}, _From, State) -> + {reply, catch run(Task), State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[Pool] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({async_submit, Task}, State) -> + try run(Task) + catch _:Error:Stacktrace -> + emqx_logger:error("[Pool] error: ~p, ~p", [Error, Stacktrace]) + end, + {noreply, State}; + +handle_cast(Msg, State) -> + emqx_logger:error("[Pool] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + emqx_logger:error("[Pool] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{pool := Pool, id := Id}) -> + gproc_pool:disconnect_worker(Pool, {Pool, Id}). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +run({M, F, A}) -> + erlang:apply(M, F, A); +run({F, A}) when is_function(F), is_list(A) -> + erlang:apply(F, A); +run(Fun) when is_function(Fun) -> + Fun(). + diff --git a/src/emqttd_pool_sup.erl b/src/emqx_pool_sup.erl similarity index 65% rename from src/emqttd_pool_sup.erl rename to src/emqx_pool_sup.erl index d5f408cd2..eb81233f9 100644 --- a/src/emqttd_pool_sup.erl +++ b/src/emqx_pool_sup.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,19 +11,13 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- -%% @doc Common Pool Supervisor --module(emqttd_pool_sup). - --author("Feng Lee "). +-module(emqx_pool_sup). -behaviour(supervisor). -%% API -export([spec/1, spec/2, start_link/3, start_link/4]). -%% Supervisor callbacks -export([init/1]). -spec(spec(list()) -> supervisor:child_spec()). @@ -33,28 +26,28 @@ spec(Args) -> -spec(spec(any(), list()) -> supervisor:child_spec()). spec(ChildId, Args) -> - {ChildId, {?MODULE, start_link, Args}, - transient, infinity, supervisor, [?MODULE]}. + #{id => ChildId, + start => {?MODULE, start_link, Args}, + restart => transient, + shutdown => infinity, + type => supervisor, + modules => [?MODULE]}. -spec(start_link(atom() | tuple(), atom(), mfa()) -> {ok, pid()} | {error, term()}). start_link(Pool, Type, MFA) -> - Schedulers = erlang:system_info(schedulers), - start_link(Pool, Type, Schedulers, MFA). + start_link(Pool, Type, emqx_vm:schedulers(), MFA). --spec(start_link(atom(), atom(), pos_integer(), mfa()) -> {ok, pid()} | {error, term()}). +-spec(start_link(atom() | tuple(), atom(), pos_integer(), mfa()) + -> {ok, pid()} | {error, term()}). start_link(Pool, Type, Size, MFA) -> supervisor:start_link(?MODULE, [Pool, Type, Size, MFA]). -%% sup_name(Pool) when is_atom(Pool) -> -%% list_to_atom(atom_to_list(Pool) ++ "_pool_sup"). - init([Pool, Type, Size, {M, F, Args}]) -> ensure_pool(Pool, Type, [{size, Size}]), {ok, {{one_for_one, 10, 3600}, [ begin ensure_pool_worker(Pool, {Pool, I}, I), - {{M, I}, {M, F, [Pool, I | Args]}, - transient, 5000, worker, [M]} + {{M, I}, {M, F, [Pool, I | Args]}, transient, 5000, worker, [M]} end || I <- lists:seq(1, Size)]}}. ensure_pool(Pool, Type, Opts) -> diff --git a/src/priority_queue.erl b/src/emqx_pqueue.erl similarity index 90% rename from src/priority_queue.erl rename to src/emqx_pqueue.erl index 7147b0bb4..f9392d5f6 100644 --- a/src/priority_queue.erl +++ b/src/emqx_pqueue.erl @@ -37,46 +37,27 @@ %% calls into the same function knowing that ordinary queues represent %% a base case. --module(priority_queue). +-module(emqx_pqueue). -export([new/0, is_queue/1, is_empty/1, len/1, plen/2, to_list/1, from_list/1, in/2, in/3, out/1, out/2, out_p/1, join/2, filter/2, fold/3, highest/1]). %%---------------------------------------------------------------------------- --ifdef(use_specs). - --type(q() :: pqueue()). -type(priority() :: integer() | 'infinity'). -type(squeue() :: {queue, [any()], [any()], non_neg_integer()}). -type(pqueue() :: squeue() | {pqueue, [{priority(), squeue()}]}). +-type(q() :: pqueue()). -export_type([q/0]). --spec(new/0 :: () -> pqueue()). --spec(is_queue/1 :: (any()) -> boolean()). --spec(is_empty/1 :: (pqueue()) -> boolean()). --spec(len/1 :: (pqueue()) -> non_neg_integer()). --spec(plen/2 :: (priority(), pqueue()) -> non_neg_integer()). --spec(to_list/1 :: (pqueue()) -> [{priority(), any()}]). --spec(from_list/1 :: ([{priority(), any()}]) -> pqueue()). --spec(in/2 :: (any(), pqueue()) -> pqueue()). --spec(in/3 :: (any(), priority(), pqueue()) -> pqueue()). --spec(out/1 :: (pqueue()) -> {empty | {value, any()}, pqueue()}). --spec(out_p/1 :: (pqueue()) -> {empty | {value, any(), priority()}, pqueue()}). --spec(join/2 :: (pqueue(), pqueue()) -> pqueue()). --spec(filter/2 :: (fun ((any()) -> boolean()), pqueue()) -> pqueue()). --spec(fold/3 :: - (fun ((any(), priority(), A) -> A), A, pqueue()) -> A). --spec(highest/1 :: (pqueue()) -> priority() | 'empty'). - --endif. - %%---------------------------------------------------------------------------- +-spec(new() -> pqueue()). new() -> {queue, [], [], 0}. +-spec(is_queue(any()) -> boolean()). is_queue({queue, R, F, L}) when is_list(R), is_list(F), is_integer(L) -> true; is_queue({pqueue, Queues}) when is_list(Queues) -> @@ -86,16 +67,19 @@ is_queue({pqueue, Queues}) when is_list(Queues) -> is_queue(_) -> false. +-spec(is_empty(pqueue()) -> boolean()). is_empty({queue, [], [], 0}) -> true; is_empty(_) -> false. +-spec(len(pqueue()) -> non_neg_integer()). len({queue, _R, _F, L}) -> L; len({pqueue, Queues}) -> lists:sum([len(Q) || {_, Q} <- Queues]). +-spec(plen(priority(), pqueue()) -> non_neg_integer()). plen(0, {queue, _R, _F, L}) -> L; plen(_, {queue, _R, _F, _}) -> @@ -106,18 +90,22 @@ plen(P, {pqueue, Queues}) -> false -> 0 end. +-spec(to_list(pqueue()) -> [{priority(), any()}]). to_list({queue, In, Out, _Len}) when is_list(In), is_list(Out) -> [{0, V} || V <- Out ++ lists:reverse(In, [])]; to_list({pqueue, Queues}) -> [{maybe_negate_priority(P), V} || {P, Q} <- Queues, {0, V} <- to_list(Q)]. +-spec(from_list([{priority(), any()}]) -> pqueue()). from_list(L) -> lists:foldl(fun ({P, E}, Q) -> in(E, P, Q) end, new(), L). +-spec(in(any(), pqueue()) -> pqueue()). in(Item, Q) -> in(Item, 0, Q). +-spec(in(any(), priority(), pqueue()) -> pqueue()). in(X, 0, {queue, [_] = In, [], 1}) -> {queue, [X], In, 2}; in(X, 0, {queue, In, Out, Len}) when is_list(In), is_list(Out) -> @@ -143,6 +131,7 @@ in(X, Priority, {pqueue, Queues}) -> end end}. +-spec(out(pqueue()) -> {empty | {value, any()}, pqueue()}). out({queue, [], [], 0} = Q) -> {empty, Q}; out({queue, [V], [], 1}) -> @@ -166,6 +155,7 @@ out({pqueue, [{P, Q} | Queues]}) -> end, {R, NewQ}. +-spec(out_p(pqueue()) -> {empty | {value, any(), priority()}, pqueue()}). out_p({queue, _, _, _} = Q) -> add_p(out(Q), 0); out_p({pqueue, [{P, _} | _]} = Q) -> add_p(out(Q), maybe_negate_priority(P)). @@ -196,6 +186,7 @@ add_p(R, P) -> case R of {{value, V}, Q} -> {{value, V, P}, Q} end. +-spec(join(pqueue(), pqueue()) -> pqueue()). join(A, {queue, [], [], 0}) -> A; join({queue, [], [], 0}, B) -> @@ -234,6 +225,7 @@ merge([{PA, A}|As], Bs = [{PB, _}|_], Acc) when PA < PB orelse PA == infinity -> merge(As = [{_, _}|_], [{PB, B}|Bs], Acc) -> merge(As, Bs, [ {PB, B} | Acc ]). +-spec(filter(fun ((any()) -> boolean()), pqueue()) -> pqueue()). filter(Pred, Q) -> fold(fun(V, P, Acc) -> case Pred(V) of true -> in(V, P, Acc); @@ -241,11 +233,13 @@ filter(Pred, Q) -> fold(fun(V, P, Acc) -> end end, new(), Q). +-spec(fold(fun ((any(), priority(), A) -> A), A, pqueue()) -> A). fold(Fun, Init, Q) -> case out_p(Q) of {empty, _Q} -> Init; {{value, V, P}, Q1} -> fold(Fun, Fun(V, P, Init), Q1) end. +-spec(highest(pqueue()) -> priority() | 'empty'). highest({queue, [], [], 0}) -> empty; highest({queue, _, _, _}) -> 0; highest({pqueue, [{P, _} | _]}) -> maybe_negate_priority(P). @@ -257,3 +251,4 @@ r2f([X,Y|R], L) -> {queue, [X,Y], lists:reverse(R, []), L}. maybe_negate_priority(infinity) -> infinity; maybe_negate_priority(P) -> -P. + diff --git a/src/emqx_protocol.erl b/src/emqx_protocol.erl new file mode 100644 index 000000000..13eff95bf --- /dev/null +++ b/src/emqx_protocol.erl @@ -0,0 +1,982 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_protocol). + +-define(LOG_HEADER, "[MQTT]"). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). +-include("logger.hrl"). + +-export([init/2]). +-export([info/1]). +-export([attrs/1]). +-export([caps/1]). +-export([stats/1]). +-export([client_id/1]). +-export([credentials/1]). +-export([parser/1]). +-export([session/1]). +-export([received/2]). +-export([process_packet/2]). +-export([deliver/2]). +-export([send/2]). +-export([shutdown/2]). + +-record(pstate, { + zone, + sendfun, + peername, + peercert, + proto_ver, + proto_name, + client_id, + is_assigned, + conn_pid, + conn_props, + ack_props, + username, + session, + clean_start, + topic_aliases, + packet_size, + will_topic, + will_msg, + keepalive, + mountpoint, + is_super, + is_bridge, + enable_ban, + enable_acl, + acl_deny_action, + recv_stats, + send_stats, + connected, + connected_at, + ignore_loop, + topic_alias_maximum + }). + +-type(state() :: #pstate{}). +-export_type([state/0]). + +-ifdef(TEST). +-compile(export_all). +-compile(nowarn_export_all). +-endif. + +-define(NO_PROPS, undefined). + +%%------------------------------------------------------------------------------ +%% Init +%%------------------------------------------------------------------------------ + +-spec(init(map(), list()) -> state()). +init(#{peername := Peername, peercert := Peercert, sendfun := SendFun}, Options) -> + Zone = proplists:get_value(zone, Options), + #pstate{zone = Zone, + sendfun = SendFun, + peername = Peername, + peercert = Peercert, + proto_ver = ?MQTT_PROTO_V4, + proto_name = <<"MQTT">>, + client_id = <<>>, + is_assigned = false, + conn_pid = self(), + username = init_username(Peercert, Options), + is_super = false, + clean_start = false, + topic_aliases = #{}, + packet_size = emqx_zone:get_env(Zone, max_packet_size), + mountpoint = emqx_zone:get_env(Zone, mountpoint), + is_bridge = false, + enable_ban = emqx_zone:get_env(Zone, enable_ban, false), + enable_acl = emqx_zone:get_env(Zone, enable_acl), + acl_deny_action = emqx_zone:get_env(Zone, acl_deny_action, ignore), + recv_stats = #{msg => 0, pkt => 0}, + send_stats = #{msg => 0, pkt => 0}, + connected = false, + ignore_loop = emqx_config:get_env(mqtt_ignore_loop_deliver, false), + topic_alias_maximum = #{to_client => 0, from_client => 0}}. + +init_username(Peercert, Options) -> + case proplists:get_value(peer_cert_as_username, Options) of + cn -> esockd_peercert:common_name(Peercert); + dn -> esockd_peercert:subject(Peercert); + crt -> Peercert; + _ -> undefined + end. + +set_username(Username, PState = #pstate{username = undefined}) -> + PState#pstate{username = Username}; +set_username(_Username, PState) -> + PState. + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +info(PState = #pstate{conn_props = ConnProps, + ack_props = AckProps, + session = Session, + topic_aliases = Aliases, + will_msg = WillMsg, + enable_acl = EnableAcl}) -> + attrs(PState) ++ [{conn_props, ConnProps}, + {ack_props, AckProps}, + {session, Session}, + {topic_aliases, Aliases}, + {will_msg, WillMsg}, + {enable_acl, EnableAcl}]. + +attrs(#pstate{zone = Zone, + client_id = ClientId, + username = Username, + peername = Peername, + peercert = Peercert, + clean_start = CleanStart, + proto_ver = ProtoVer, + proto_name = ProtoName, + keepalive = Keepalive, + mountpoint = Mountpoint, + is_super = IsSuper, + is_bridge = IsBridge, + connected_at = ConnectedAt}) -> + [{zone, Zone}, + {client_id, ClientId}, + {username, Username}, + {peername, Peername}, + {peercert, Peercert}, + {proto_ver, ProtoVer}, + {proto_name, ProtoName}, + {clean_start, CleanStart}, + {keepalive, Keepalive}, + {mountpoint, Mountpoint}, + {is_super, IsSuper}, + {is_bridge, IsBridge}, + {connected_at, ConnectedAt}]. + +caps(#pstate{zone = Zone}) -> + emqx_mqtt_caps:get_caps(Zone). + +client_id(#pstate{client_id = ClientId}) -> + ClientId. + +credentials(#pstate{zone = Zone, + client_id = ClientId, + username = Username, + peername = Peername}) -> + #{zone => Zone, + client_id => ClientId, + username => Username, + peername => Peername}. + +stats(#pstate{recv_stats = #{pkt := RecvPkt, msg := RecvMsg}, + send_stats = #{pkt := SendPkt, msg := SendMsg}}) -> + [{recv_pkt, RecvPkt}, + {recv_msg, RecvMsg}, + {send_pkt, SendPkt}, + {send_msg, SendMsg}]. + +session(#pstate{session = SPid}) -> + SPid. + +parser(#pstate{packet_size = Size, proto_ver = Ver}) -> + emqx_frame:initial_state(#{max_packet_size => Size, version => Ver}). + +%%------------------------------------------------------------------------------ +%% Packet Received +%%------------------------------------------------------------------------------ + +set_protover(?CONNECT_PACKET(#mqtt_packet_connect{ + proto_ver = ProtoVer}), + PState) -> + PState#pstate{ proto_ver = ProtoVer }; +set_protover(_Packet, PState) -> + PState. + +-spec(received(emqx_mqtt_types:packet(), state()) -> + {ok, state()} | {error, term()} | {error, term(), state()} | {stop, term(), state()}). +received(?PACKET(Type), PState = #pstate{connected = false}) when Type =/= ?CONNECT -> + {error, proto_not_connected, PState}; + +received(?PACKET(?CONNECT), PState = #pstate{connected = true}) -> + {error, proto_unexpected_connect, PState}; + +received(Packet = ?PACKET(Type), PState) -> + PState1 = set_protover(Packet, PState), + trace(recv, Packet), + try emqx_packet:validate(Packet) of + true -> + case preprocess_properties(Packet, PState1) of + {error, ReasonCode} -> + {error, ReasonCode, PState1}; + {Packet1, PState2} -> + process_packet(Packet1, inc_stats(recv, Type, PState2)) + end + catch + error : protocol_error -> + deliver({disconnect, ?RC_PROTOCOL_ERROR}, PState1), + {error, protocol_error, PState}; + error : subscription_identifier_invalid -> + deliver({disconnect, ?RC_SUBSCRIPTION_IDENTIFIERS_NOT_SUPPORTED}, PState1), + {error, subscription_identifier_invalid, PState1}; + error : topic_alias_invalid -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState1), + {error, topic_alias_invalid, PState1}; + error : topic_filters_invalid -> + deliver({disconnect, ?RC_TOPIC_FILTER_INVALID}, PState1), + {error, topic_filters_invalid, PState1}; + error : topic_name_invalid -> + deliver({disconnect, ?RC_TOPIC_FILTER_INVALID}, PState1), + {error, topic_filters_invalid, PState1}; + error : Reason -> + deliver({disconnect, ?RC_MALFORMED_PACKET}, PState1), + {error, Reason, PState1} + end. + +%%------------------------------------------------------------------------------ +%% Preprocess MQTT Properties +%%------------------------------------------------------------------------------ +preprocess_properties(Packet = #mqtt_packet{ + variable = #mqtt_packet_connect{ + properties = #{'Topic-Alias-Maximum' := ToClient} + } + }, + PState = #pstate{topic_alias_maximum = TopicAliasMaximum}) -> + {Packet, PState#pstate{topic_alias_maximum = TopicAliasMaximum#{to_client => ToClient}}}; + +%% Subscription Identifier +preprocess_properties(Packet = #mqtt_packet{ + variable = Subscribe = #mqtt_packet_subscribe{ + properties = #{'Subscription-Identifier' := SubId}, + topic_filters = TopicFilters + } + }, + PState = #pstate{proto_ver = ?MQTT_PROTO_V5}) -> + TopicFilters1 = [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters], + {Packet#mqtt_packet{variable = Subscribe#mqtt_packet_subscribe{topic_filters = TopicFilters1}}, PState}; + +%% Topic Alias Mapping +preprocess_properties(#mqtt_packet{ + variable = #mqtt_packet_publish{ + properties = #{'Topic-Alias' := 0}} + }, + PState) -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID}; + +preprocess_properties(Packet = #mqtt_packet{ + variable = Publish = #mqtt_packet_publish{ + topic_name = <<>>, + properties = #{'Topic-Alias' := AliasId}} + }, + PState = #pstate{proto_ver = ?MQTT_PROTO_V5, + topic_aliases = Aliases, + topic_alias_maximum = #{from_client := TopicAliasMaximum}}) -> + case AliasId =< TopicAliasMaximum of + true -> + {Packet#mqtt_packet{variable = Publish#mqtt_packet_publish{ + topic_name = maps:get(AliasId, Aliases, <<>>)}}, PState}; + false -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID} + end; + +preprocess_properties(Packet = #mqtt_packet{ + variable = #mqtt_packet_publish{ + topic_name = Topic, + properties = #{'Topic-Alias' := AliasId}} + }, + PState = #pstate{proto_ver = ?MQTT_PROTO_V5, + topic_aliases = Aliases, + topic_alias_maximum = #{from_client := TopicAliasMaximum}}) -> + case AliasId =< TopicAliasMaximum of + true -> + {Packet, PState#pstate{topic_aliases = maps:put(AliasId, Topic, Aliases)}}; + false -> + deliver({disconnect, ?RC_TOPIC_ALIAS_INVALID}, PState), + {error, ?RC_TOPIC_ALIAS_INVALID} + end; + +preprocess_properties(Packet, PState) -> + {Packet, PState}. + +%%------------------------------------------------------------------------------ +%% Process MQTT Packet +%%------------------------------------------------------------------------------ +process_packet(?CONNECT_PACKET( + #mqtt_packet_connect{proto_name = ProtoName, + proto_ver = ProtoVer, + is_bridge = IsBridge, + clean_start = CleanStart, + keepalive = Keepalive, + properties = ConnProps, + client_id = ClientId, + username = Username, + password = Password} = ConnPkt), PState) -> + + NewClientId = maybe_use_username_as_clientid(ClientId, Username, PState), + + emqx_logger:set_metadata_client_id(NewClientId), + + %% TODO: Mountpoint... + %% Msg -> emqx_mountpoint:mount(MountPoint, Msg) + + PState1 = set_username(Username, + PState#pstate{client_id = NewClientId, + proto_ver = ProtoVer, + proto_name = ProtoName, + clean_start = CleanStart, + keepalive = Keepalive, + conn_props = ConnProps, + is_bridge = IsBridge, + connected_at = os:timestamp()}), + + connack( + case check_connect(ConnPkt, PState1) of + {ok, PState2} -> + case authenticate(credentials(PState2), Password) of + {ok, IsSuper} -> + %% Maybe assign a clientId + PState3 = maybe_assign_client_id(PState2#pstate{is_super = IsSuper, + will_msg = make_will_msg(ConnPkt)}), + emqx_logger:set_metadata_client_id(PState3#pstate.client_id), + %% Open session + case try_open_session(PState3) of + {ok, SPid, SP} -> + PState4 = PState3#pstate{session = SPid, connected = true}, + ok = emqx_cm:register_connection(client_id(PState4)), + true = emqx_cm:set_conn_attrs(client_id(PState4), attrs(PState4)), + %% Start keepalive + start_keepalive(Keepalive, PState4), + %% Success + {?RC_SUCCESS, SP, PState4}; + {error, Error} -> + ?LOG(error, "Failed to open session: ~p", [Error]), + {?RC_UNSPECIFIED_ERROR, PState1} + end; + {error, Reason} -> + ?LOG(error, "Username '~s' login failed for ~p", [Username, Reason]), + {?RC_NOT_AUTHORIZED, PState1} + end; + {error, ReasonCode} -> + {ReasonCode, PState1} + end); + +process_packet(Packet = ?PUBLISH_PACKET(?QOS_0, Topic, _PacketId, _Payload), PState) -> + case check_publish(Packet, PState) of + {ok, PState1} -> + do_publish(Packet, PState1); + {error, ReasonCode} -> + ?LOG(warning, "Cannot publish qos0 message to ~s for ~s", + [Topic, emqx_reason_codes:text(ReasonCode)]), + do_acl_deny_action(Packet, ReasonCode, PState) + end; + +process_packet(Packet = ?PUBLISH_PACKET(?QOS_1, Topic, PacketId, _Payload), PState) -> + case check_publish(Packet, PState) of + {ok, PState1} -> + do_publish(Packet, PState1); + {error, ReasonCode} -> + ?LOG(warning, "Cannot publish qos1 message to ~s for ~s", + [Topic, emqx_reason_codes:text(ReasonCode)]), + case deliver({puback, PacketId, ReasonCode}, PState) of + {ok, _PState} -> + do_acl_deny_action(Packet, ReasonCode, PState); + Error -> + Error + end + end; + +process_packet(Packet = ?PUBLISH_PACKET(?QOS_2, Topic, PacketId, _Payload), PState) -> + case check_publish(Packet, PState) of + {ok, PState1} -> + do_publish(Packet, PState1); + {error, ReasonCode} -> + ?LOG(warning, "Cannot publish qos2 message to ~s for ~s", + [Topic, emqx_reason_codes:text(ReasonCode)]), + case deliver({pubrec, PacketId, ?RC_NOT_AUTHORIZED}, PState) of + {ok, _PState} -> + do_acl_deny_action(Packet, ReasonCode, PState); + Error -> + Error + end + end; + +process_packet(?PUBACK_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> + {ok = emqx_session:puback(SPid, PacketId, ReasonCode), PState}; + +process_packet(?PUBREC_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> + case emqx_session:pubrec(SPid, PacketId, ReasonCode) of + ok -> + send(?PUBREL_PACKET(PacketId), PState); + {error, NotFound} -> + send(?PUBREL_PACKET(PacketId, NotFound), PState) + end; + +process_packet(?PUBREL_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> + case emqx_session:pubrel(SPid, PacketId, ReasonCode) of + ok -> + send(?PUBCOMP_PACKET(PacketId), PState); + {error, NotFound} -> + send(?PUBCOMP_PACKET(PacketId, NotFound), PState) + end; + +process_packet(?PUBCOMP_PACKET(PacketId, ReasonCode), PState = #pstate{session = SPid}) -> + {ok = emqx_session:pubcomp(SPid, PacketId, ReasonCode), PState}; + +process_packet(Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), + PState = #pstate{session = SPid, mountpoint = Mountpoint, + proto_ver = ProtoVer, is_bridge = IsBridge, + ignore_loop = IgnoreLoop}) -> + RawTopicFilters1 = if ProtoVer < ?MQTT_PROTO_V5 -> + IfIgnoreLoop = case IgnoreLoop of true -> 1; false -> 0 end, + case IsBridge of + true -> [{RawTopic, SubOpts#{rap => 1, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters]; + false -> [{RawTopic, SubOpts#{rap => 0, nl => IfIgnoreLoop}} || {RawTopic, SubOpts} <- RawTopicFilters] + end; + true -> + RawTopicFilters + end, + case check_subscribe( + parse_topic_filters(?SUBSCRIBE, RawTopicFilters1), PState) of + {ok, TopicFilters} -> + case emqx_hooks:run('client.subscribe', [credentials(PState)], TopicFilters) of + {ok, TopicFilters1} -> + ok = emqx_session:subscribe(SPid, PacketId, Properties, + emqx_mountpoint:mount(Mountpoint, TopicFilters1)), + {ok, PState}; + {stop, _} -> + ReasonCodes = lists:duplicate(length(TopicFilters), + ?RC_IMPLEMENTATION_SPECIFIC_ERROR), + deliver({suback, PacketId, ReasonCodes}, PState) + end; + {error, TopicFilters} -> + {ReverseSubTopics, ReverseReasonCodes} = + lists:foldl(fun({Topic, #{rc := ?RC_SUCCESS}}, {Topics, Codes}) -> + {[Topic|Topics], [?RC_IMPLEMENTATION_SPECIFIC_ERROR | Codes]}; + ({Topic, #{rc := Code}}, {Topics, Codes}) -> + {[Topic|Topics], [Code|Codes]} + end, {[], []}, TopicFilters), + {SubTopics, ReasonCodes} = {lists:reverse(ReverseSubTopics), lists:reverse(ReverseReasonCodes)}, + ?LOG(warning, "Cannot subscribe ~p for ~p", + [SubTopics, [emqx_reason_codes:text(R) || R <- ReasonCodes]]), + deliver({suback, PacketId, ReasonCodes}, PState), + do_acl_deny_action(Packet, ReasonCodes, PState) + end; + +process_packet(?UNSUBSCRIBE_PACKET(PacketId, Properties, RawTopicFilters), + PState = #pstate{session = SPid, mountpoint = MountPoint}) -> + case emqx_hooks:run('client.unsubscribe', [credentials(PState)], + parse_topic_filters(?UNSUBSCRIBE, RawTopicFilters)) of + {ok, TopicFilters} -> + ok = emqx_session:unsubscribe(SPid, PacketId, Properties, + emqx_mountpoint:mount(MountPoint, TopicFilters)), + {ok, PState}; + {stop, _Acc} -> + ReasonCodes = lists:duplicate(length(RawTopicFilters), + ?RC_IMPLEMENTATION_SPECIFIC_ERROR), + deliver({unsuback, PacketId, ReasonCodes}, PState) + end; + +process_packet(?PACKET(?PINGREQ), PState) -> + send(?PACKET(?PINGRESP), PState); + +process_packet(?DISCONNECT_PACKET(?RC_SUCCESS, #{'Session-Expiry-Interval' := Interval}), + PState = #pstate{session = SPid, conn_props = #{'Session-Expiry-Interval' := OldInterval}}) -> + case Interval =/= 0 andalso OldInterval =:= 0 of + true -> + deliver({disconnect, ?RC_PROTOCOL_ERROR}, PState), + {error, protocol_error, PState#pstate{will_msg = undefined}}; + false -> + emqx_session:update_expiry_interval(SPid, Interval), + %% Clean willmsg + {stop, normal, PState#pstate{will_msg = undefined}} + end; +process_packet(?DISCONNECT_PACKET(?RC_SUCCESS), PState) -> + {stop, normal, PState#pstate{will_msg = undefined}}; +process_packet(?DISCONNECT_PACKET(_), PState) -> + {stop, normal, PState}. + +%%------------------------------------------------------------------------------ +%% ConnAck --> Client +%%------------------------------------------------------------------------------ + +connack({?RC_SUCCESS, SP, PState}) -> + emqx_hooks:run('client.connected', [credentials(PState), ?RC_SUCCESS, attrs(PState)]), + deliver({connack, ?RC_SUCCESS, sp(SP)}, update_mountpoint(PState)); + +connack({ReasonCode, PState = #pstate{proto_ver = ProtoVer}}) -> + emqx_hooks:run('client.connected', [credentials(PState), ReasonCode, attrs(PState)]), + [ReasonCode1] = reason_codes_compat(connack, [ReasonCode], ProtoVer), + _ = deliver({connack, ReasonCode1}, PState), + {error, emqx_reason_codes:name(ReasonCode1, ProtoVer), PState}. + +%%------------------------------------------------------------------------------ +%% Publish Message -> Broker +%%------------------------------------------------------------------------------ + +do_publish(Packet = ?PUBLISH_PACKET(QoS, PacketId), + PState = #pstate{session = SPid, mountpoint = MountPoint}) -> + Msg = emqx_mountpoint:mount(MountPoint, + emqx_packet:to_message(credentials(PState), Packet)), + puback(QoS, PacketId, emqx_session:publish(SPid, PacketId, Msg), PState). + +%%------------------------------------------------------------------------------ +%% Puback -> Client +%%------------------------------------------------------------------------------ + +puback(?QOS_0, _PacketId, _Result, PState) -> + {ok, PState}; +puback(?QOS_1, PacketId, [], PState) -> + deliver({puback, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); +puback(?QOS_1, PacketId, [_|_], PState) -> %%TODO: check the dispatch? + deliver({puback, PacketId, ?RC_SUCCESS}, PState); +puback(?QOS_1, PacketId, {error, ReasonCode}, PState) -> + deliver({puback, PacketId, ReasonCode}, PState); +puback(?QOS_2, PacketId, [], PState) -> + deliver({pubrec, PacketId, ?RC_NO_MATCHING_SUBSCRIBERS}, PState); +puback(?QOS_2, PacketId, [_|_], PState) -> %%TODO: check the dispatch? + deliver({pubrec, PacketId, ?RC_SUCCESS}, PState); +puback(?QOS_2, PacketId, {error, ReasonCode}, PState) -> + deliver({pubrec, PacketId, ReasonCode}, PState). + +%%------------------------------------------------------------------------------ +%% Deliver Packet -> Client +%%------------------------------------------------------------------------------ + +-spec(deliver(tuple(), state()) -> {ok, state()} | {error, term()}). +deliver({connack, ReasonCode}, PState) -> + send(?CONNACK_PACKET(ReasonCode), PState); + +deliver({connack, ?RC_SUCCESS, SP}, PState = #pstate{zone = Zone, + proto_ver = ?MQTT_PROTO_V5, + client_id = ClientId, + conn_props = ConnProps, + is_assigned = IsAssigned, + topic_alias_maximum = TopicAliasMaximum}) -> + ResponseInformation = case maps:find('Request-Response-Information', ConnProps) of + {ok, 1} -> + iolist_to_binary(emqx_config:get_env(response_topic_prefix)); + _ -> + <<>> + end, + #{max_packet_size := MaxPktSize, + max_qos_allowed := MaxQoS, + mqtt_retain_available := Retain, + max_topic_alias := MaxAlias, + mqtt_shared_subscription := Shared, + mqtt_wildcard_subscription := Wildcard} = caps(PState), + Props = #{'Retain-Available' => flag(Retain), + 'Maximum-Packet-Size' => MaxPktSize, + 'Topic-Alias-Maximum' => MaxAlias, + 'Wildcard-Subscription-Available' => flag(Wildcard), + 'Subscription-Identifier-Available' => 1, + 'Response-Information' => ResponseInformation, + 'Shared-Subscription-Available' => flag(Shared)}, + + Props1 = if + MaxQoS =:= ?QOS_2 -> + Props; + true -> + maps:put('Maximum-QoS', MaxQoS, Props) + end, + + Props2 = if IsAssigned -> + Props1#{'Assigned-Client-Identifier' => ClientId}; + true -> Props1 + + end, + + Props3 = case emqx_zone:get_env(Zone, server_keepalive) of + undefined -> Props2; + Keepalive -> Props2#{'Server-Keep-Alive' => Keepalive} + end, + + PState1 = PState#pstate{topic_alias_maximum = TopicAliasMaximum#{from_client => MaxAlias}}, + + send(?CONNACK_PACKET(?RC_SUCCESS, SP, Props3), PState1); + +deliver({connack, ReasonCode, SP}, PState) -> + send(?CONNACK_PACKET(ReasonCode, SP), PState); + +deliver({publish, PacketId, Msg}, PState = #pstate{mountpoint = MountPoint}) -> + _ = emqx_hooks:run('message.delivered', [credentials(PState)], Msg), + Msg1 = emqx_message:update_expiry(Msg), + Msg2 = emqx_mountpoint:unmount(MountPoint, Msg1), + send(emqx_packet:from_message(PacketId, emqx_message:remove_topic_alias(Msg2)), PState); + +deliver({puback, PacketId, ReasonCode}, PState) -> + send(?PUBACK_PACKET(PacketId, ReasonCode), PState); + +deliver({pubrel, PacketId}, PState) -> + send(?PUBREL_PACKET(PacketId), PState); + +deliver({pubrec, PacketId, ReasonCode}, PState) -> + send(?PUBREC_PACKET(PacketId, ReasonCode), PState); + +deliver({suback, PacketId, ReasonCodes}, PState = #pstate{proto_ver = ProtoVer}) -> + send(?SUBACK_PACKET(PacketId, reason_codes_compat(suback, ReasonCodes, ProtoVer)), PState); + +deliver({unsuback, PacketId, ReasonCodes}, PState = #pstate{proto_ver = ProtoVer}) -> + send(?UNSUBACK_PACKET(PacketId, reason_codes_compat(unsuback, ReasonCodes, ProtoVer)), PState); + +%% Deliver a disconnect for mqtt 5.0 +deliver({disconnect, ReasonCode}, PState = #pstate{proto_ver = ?MQTT_PROTO_V5}) -> + send(?DISCONNECT_PACKET(ReasonCode), PState); + +deliver({disconnect, _ReasonCode}, PState) -> + {ok, PState}. + +%%------------------------------------------------------------------------------ +%% Send Packet to Client + +-spec(send(emqx_mqtt_types:packet(), state()) -> {ok, state()} | {error, term()}). +send(Packet = ?PACKET(Type), PState = #pstate{proto_ver = Ver, sendfun = SendFun}) -> + trace(send, Packet), + case SendFun(Packet, #{version => Ver}) of + ok -> + emqx_metrics:sent(Packet), + {ok, inc_stats(send, Type, PState)}; + {error, Reason} -> + {error, Reason} + end. + +%%------------------------------------------------------------------------------ +%% Maybe use username replace client id + +maybe_use_username_as_clientid(ClientId, undefined, _PState) -> + ClientId; +maybe_use_username_as_clientid(ClientId, Username, #pstate{zone = Zone}) -> + case emqx_zone:get_env(Zone, use_username_as_clientid, false) of + true -> Username; + false -> ClientId + end. + +%%------------------------------------------------------------------------------ +%% Assign a clientId + +maybe_assign_client_id(PState = #pstate{client_id = <<>>, ack_props = AckProps}) -> + ClientId = emqx_guid:to_base62(emqx_guid:gen()), + AckProps1 = set_property('Assigned-Client-Identifier', ClientId, AckProps), + PState#pstate{client_id = ClientId, is_assigned = true, ack_props = AckProps1}; +maybe_assign_client_id(PState) -> + PState. + +try_open_session(PState = #pstate{zone = Zone, + client_id = ClientId, + conn_pid = ConnPid, + username = Username, + clean_start = CleanStart, + will_msg = WillMsg}) -> + + SessAttrs = #{ + zone => Zone, + client_id => ClientId, + conn_pid => ConnPid, + username => Username, + clean_start => CleanStart, + will_msg => WillMsg + }, + SessAttrs1 = lists:foldl(fun set_session_attrs/2, SessAttrs, [{max_inflight, PState}, {expiry_interval, PState}]), + case emqx_sm:open_session(SessAttrs1) of + {ok, SPid} -> + {ok, SPid, false}; + Other -> Other + end. + + +set_session_attrs({max_inflight, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) -> + maps:put(max_inflight, get_property('Receive-Maximum', ConnProps, 65535), SessAttrs); + +set_session_attrs({max_inflight, #pstate{zone = Zone}}, SessAttrs) -> + maps:put(max_inflight, emqx_zone:get_env(Zone, max_inflight, 65535), SessAttrs); + +set_session_attrs({expiry_interval, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) -> + maps:put(expiry_interval, get_property('Session-Expiry-Interval', ConnProps, 0), SessAttrs); + +set_session_attrs({expiry_interval, #pstate{zone = Zone, clean_start = CleanStart}}, SessAttrs) -> + maps:put(expiry_interval, case CleanStart of + true -> 0; + false -> emqx_zone:get_env(Zone, session_expiry_interval, 16#ffffffff) + end, SessAttrs); + +set_session_attrs({topic_alias_maximum, #pstate{proto_ver = ?MQTT_PROTO_V5, conn_props = ConnProps}}, SessAttrs) -> + maps:put(topic_alias_maximum, get_property('Topic-Alias-Maximum', ConnProps, 0), SessAttrs); + +set_session_attrs({topic_alias_maximum, #pstate{zone = Zone}}, SessAttrs) -> + maps:put(topic_alias_maximum, emqx_zone:get_env(Zone, max_topic_alias, 0), SessAttrs); + +set_session_attrs(_, SessAttrs) -> + SessAttrs. + +authenticate(Credentials, Password) -> + case emqx_access_control:authenticate(Credentials, Password) of + ok -> {ok, false}; + {ok, IsSuper} when is_boolean(IsSuper) -> + {ok, IsSuper}; + {ok, Result} when is_map(Result) -> + {ok, maps:get(is_superuser, Result, false)}; + {error, Error} -> + {error, Error} + end. + +set_property(Name, Value, ?NO_PROPS) -> + #{Name => Value}; +set_property(Name, Value, Props) -> + Props#{Name => Value}. + +get_property(_Name, undefined, Default) -> + Default; +get_property(Name, Props, Default) -> + maps:get(Name, Props, Default). + +make_will_msg(#mqtt_packet_connect{proto_ver = ProtoVer, + will_props = WillProps} = ConnPkt) -> + emqx_packet:will_msg( + case ProtoVer of + ?MQTT_PROTO_V5 -> + WillDelayInterval = get_property('Will-Delay-Interval', WillProps, 0), + ConnPkt#mqtt_packet_connect{ + will_props = set_property('Will-Delay-Interval', WillDelayInterval, WillProps)}; + _ -> + ConnPkt + end). + +%%------------------------------------------------------------------------------ +%% Check Packet +%%------------------------------------------------------------------------------ + +check_connect(Packet, PState) -> + run_check_steps([fun check_proto_ver/2, + fun check_client_id/2, + fun check_banned/2, + fun check_will_topic/2], Packet, PState). + +check_proto_ver(#mqtt_packet_connect{proto_ver = Ver, + proto_name = Name}, _PState) -> + case lists:member({Ver, Name}, ?PROTOCOL_NAMES) of + true -> ok; + false -> {error, ?RC_PROTOCOL_ERROR} + end. + +%% MQTT3.1 does not allow null clientId +check_client_id(#mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V3, + client_id = <<>>}, _PState) -> + {error, ?RC_CLIENT_IDENTIFIER_NOT_VALID}; + +%% Issue#599: Null clientId and clean_start = false +check_client_id(#mqtt_packet_connect{client_id = <<>>, + clean_start = false}, _PState) -> + {error, ?RC_CLIENT_IDENTIFIER_NOT_VALID}; + +check_client_id(#mqtt_packet_connect{client_id = <<>>, + clean_start = true}, _PState) -> + ok; + +check_client_id(#mqtt_packet_connect{client_id = ClientId}, #pstate{zone = Zone}) -> + Len = byte_size(ClientId), + MaxLen = emqx_zone:get_env(Zone, max_clientid_len), + case (1 =< Len) andalso (Len =< MaxLen) of + true -> ok; + false -> {error, ?RC_CLIENT_IDENTIFIER_NOT_VALID} + end. + +check_banned(_ConnPkt, #pstate{enable_ban = false}) -> + ok; +check_banned(#mqtt_packet_connect{client_id = ClientId, username = Username}, + #pstate{peername = Peername}) -> + case emqx_banned:check(#{client_id => ClientId, + username => Username, + peername => Peername}) of + true -> {error, ?RC_BANNED}; + false -> ok + end. + +check_will_topic(#mqtt_packet_connect{will_flag = false}, _PState) -> + ok; +check_will_topic(#mqtt_packet_connect{will_topic = WillTopic} = ConnPkt, PState) -> + try emqx_topic:validate(WillTopic) of + true -> check_will_acl(ConnPkt, PState) + catch error : _Error -> + {error, ?RC_TOPIC_NAME_INVALID} + end. + +check_will_acl(_ConnPkt, #pstate{enable_acl = EnableAcl}) + when not EnableAcl -> + ok; +check_will_acl(#mqtt_packet_connect{will_topic = WillTopic}, PState) -> + case emqx_access_control:check_acl(credentials(PState), publish, WillTopic) of + allow -> ok; + deny -> + ?LOG(warning, "Cannot publish will message to ~p for acl checking failed", [WillTopic]), + {error, ?RC_UNSPECIFIED_ERROR} + end. + +check_publish(Packet, PState) -> + run_check_steps([fun check_pub_caps/2, + fun check_pub_acl/2], Packet, PState). + +check_pub_caps(#mqtt_packet{header = #mqtt_packet_header{qos = QoS, retain = Retain}, + variable = #mqtt_packet_publish{ properties = _Properties}}, + #pstate{zone = Zone}) -> + emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain}). + +check_pub_acl(_Packet, #pstate{is_super = IsSuper, enable_acl = EnableAcl}) + when IsSuper orelse (not EnableAcl) -> + ok; + +check_pub_acl(#mqtt_packet{variable = #mqtt_packet_publish{topic_name = Topic}}, PState) -> + case emqx_access_control:check_acl(credentials(PState), publish, Topic) of + allow -> ok; + deny -> + {error, ?RC_NOT_AUTHORIZED} + end. + +run_check_steps([], _Packet, PState) -> + {ok, PState}; +run_check_steps([Check|Steps], Packet, PState) -> + case Check(Packet, PState) of + ok -> + run_check_steps(Steps, Packet, PState); + {ok, PState1} -> + run_check_steps(Steps, Packet, PState1); + Error = {error, _RC} -> + Error + end. + +check_subscribe(TopicFilters, PState = #pstate{zone = Zone}) -> + case emqx_mqtt_caps:check_sub(Zone, TopicFilters) of + {ok, TopicFilter1} -> + check_sub_acl(TopicFilter1, PState); + {error, TopicFilter1} -> + {error, TopicFilter1} + end. + +check_sub_acl(TopicFilters, #pstate{is_super = IsSuper, enable_acl = EnableAcl}) + when IsSuper orelse (not EnableAcl) -> + {ok, TopicFilters}; + +check_sub_acl(TopicFilters, PState) -> + Credentials = credentials(PState), + lists:foldr( + fun({Topic, SubOpts}, {Ok, Acc}) -> + case emqx_access_control:check_acl(Credentials, subscribe, Topic) of + allow -> {Ok, [{Topic, SubOpts}|Acc]}; + deny -> + {error, [{Topic, SubOpts#{rc := ?RC_NOT_AUTHORIZED}}|Acc]} + end + end, {ok, []}, TopicFilters). + +trace(recv, Packet) -> + ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]); +trace(send, Packet) -> + ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)]). + +inc_stats(recv, Type, PState = #pstate{recv_stats = Stats}) -> + PState#pstate{recv_stats = inc_stats(Type, Stats)}; + +inc_stats(send, Type, PState = #pstate{send_stats = Stats}) -> + PState#pstate{send_stats = inc_stats(Type, Stats)}. + +inc_stats(Type, Stats = #{pkt := PktCnt, msg := MsgCnt}) -> + Stats#{pkt := PktCnt + 1, msg := case Type =:= ?PUBLISH of + true -> MsgCnt + 1; + false -> MsgCnt + end}. + +shutdown(_Reason, #pstate{client_id = undefined}) -> + ok; +shutdown(_Reason, #pstate{connected = false}) -> + ok; +shutdown(conflict, _PState) -> + ok; +shutdown(discard, _PState) -> + ok; +shutdown(Reason, PState) -> + ?LOG(info, "Shutdown for ~p", [Reason]), + emqx_hooks:run('client.disconnected', [credentials(PState), Reason]). + +start_keepalive(0, _PState) -> + ignore; +start_keepalive(Secs, #pstate{zone = Zone}) when Secs > 0 -> + Backoff = emqx_zone:get_env(Zone, keepalive_backoff, 0.75), + self() ! {keepalive, start, round(Secs * Backoff)}. + +%%----------------------------------------------------------------------------- +%% Parse topic filters +%%----------------------------------------------------------------------------- + +parse_topic_filters(?SUBSCRIBE, RawTopicFilters) -> + [emqx_topic:parse(RawTopic, SubOpts) || {RawTopic, SubOpts} <- RawTopicFilters]; + +parse_topic_filters(?UNSUBSCRIBE, RawTopicFilters) -> + lists:map(fun emqx_topic:parse/1, RawTopicFilters). + +%%------------------------------------------------------------------------------ +%% Update mountpoint + +update_mountpoint(PState = #pstate{mountpoint = undefined}) -> + PState; +update_mountpoint(PState = #pstate{mountpoint = MountPoint}) -> + PState#pstate{mountpoint = emqx_mountpoint:replvar(MountPoint, credentials(PState))}. + +sp(true) -> 1; +sp(false) -> 0. + +flag(false) -> 0; +flag(true) -> 1. + +%%------------------------------------------------------------------------------ +%% Execute actions in case acl deny + +do_acl_deny_action(?PUBLISH_PACKET(?QOS_0, _Topic, _PacketId, _Payload), + ?RC_NOT_AUTHORIZED, PState = #pstate{proto_ver = ProtoVer, + acl_deny_action = disconnect}) -> + {error, emqx_reason_codes:name(?RC_NOT_AUTHORIZED, ProtoVer), PState}; + +do_acl_deny_action(?PUBLISH_PACKET(?QOS_1, _Topic, _PacketId, _Payload), + ?RC_NOT_AUTHORIZED, PState = #pstate{proto_ver = ProtoVer, + acl_deny_action = disconnect}) -> + deliver({disconnect, ?RC_NOT_AUTHORIZED}, PState), + {error, emqx_reason_codes:name(?RC_NOT_AUTHORIZED, ProtoVer), PState}; + +do_acl_deny_action(?PUBLISH_PACKET(?QOS_2, _Topic, _PacketId, _Payload), + ?RC_NOT_AUTHORIZED, PState = #pstate{proto_ver = ProtoVer, + acl_deny_action = disconnect}) -> + deliver({disconnect, ?RC_NOT_AUTHORIZED}, PState), + {error, emqx_reason_codes:name(?RC_NOT_AUTHORIZED, ProtoVer), PState}; + +do_acl_deny_action(?SUBSCRIBE_PACKET(_PacketId, _Properties, _RawTopicFilters), + ReasonCodes, PState = #pstate{proto_ver = ProtoVer, + acl_deny_action = disconnect}) -> + case lists:member(?RC_NOT_AUTHORIZED, ReasonCodes) of + true -> + deliver({disconnect, ?RC_NOT_AUTHORIZED}, PState), + {error, emqx_reason_codes:name(?RC_NOT_AUTHORIZED, ProtoVer), PState}; + false -> + {ok, PState} + end; +do_acl_deny_action(_PubSupPacket, _ReasonCode, PState) -> + {ok, PState}. + +%% Reason code compat +reason_codes_compat(_PktType, ReasonCodes, ?MQTT_PROTO_V5) -> + ReasonCodes; +reason_codes_compat(unsuback, _ReasonCodes, _ProtoVer) -> + undefined; +reason_codes_compat(PktType, ReasonCodes, _ProtoVer) -> + [emqx_reason_codes:compat(PktType, RC) || RC <- ReasonCodes]. diff --git a/src/emqx_rate_limiter.erl b/src/emqx_rate_limiter.erl new file mode 100644 index 000000000..c8145dbbc --- /dev/null +++ b/src/emqx_rate_limiter.erl @@ -0,0 +1,65 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_rate_limiter). + +-behaviour(gen_server). + +%% API +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, {}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% @doc Starts the server +-spec(start_link() -> {ok, pid()} | ignore | {error, any()}). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +init([]) -> + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + diff --git a/src/emqx_reason_codes.erl b/src/emqx_reason_codes.erl new file mode 100644 index 000000000..5b7f7e42f --- /dev/null +++ b/src/emqx_reason_codes.erl @@ -0,0 +1,145 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc MQTT5 reason codes +-module(emqx_reason_codes). + +-include("emqx_mqtt.hrl"). + +-export([name/2, text/1]). +-export([compat/2]). + +name(I, Ver) when Ver >= ?MQTT_PROTO_V5 -> + name(I); +name(0, _Ver) -> connection_acceptd; +name(1, _Ver) -> unacceptable_protocol_version; +name(2, _Ver) -> client_identifier_not_valid; +name(3, _Ver) -> server_unavaliable; +name(4, _Ver) -> malformed_username_or_password; +name(5, _Ver) -> unauthorized_client; +name(_, _Ver) -> unknown_error. + +name(16#00) -> success; +name(16#01) -> granted_qos1; +name(16#02) -> granted_qos2; +name(16#04) -> disconnect_with_will_message; +name(16#10) -> no_matching_subscribers; +name(16#11) -> no_subscription_existed; +name(16#18) -> continue_authentication; +name(16#19) -> re_authenticate; +name(16#80) -> unspecified_error; +name(16#81) -> malformed_Packet; +name(16#82) -> protocol_error; +name(16#83) -> implementation_specific_error; +name(16#84) -> unsupported_protocol_version; +name(16#85) -> client_identifier_not_valid; +name(16#86) -> bad_username_or_password; +name(16#87) -> not_authorized; +name(16#88) -> server_unavailable; +name(16#89) -> server_busy; +name(16#8A) -> banned; +name(16#8B) -> server_shutting_down; +name(16#8C) -> bad_authentication_method; +name(16#8D) -> keepalive_timeout; +name(16#8E) -> session_taken_over; +name(16#8F) -> topic_filter_invalid; +name(16#90) -> topic_name_invalid; +name(16#91) -> packet_identifier_inuse; +name(16#92) -> packet_identifier_not_found; +name(16#93) -> receive_maximum_exceeded; +name(16#94) -> topic_alias_invalid; +name(16#95) -> packet_too_large; +name(16#96) -> message_rate_too_high; +name(16#97) -> quota_exceeded; +name(16#98) -> administrative_action; +name(16#99) -> payload_format_invalid; +name(16#9A) -> retain_not_supported; +name(16#9B) -> qos_not_supported; +name(16#9C) -> use_another_server; +name(16#9D) -> server_moved; +name(16#9E) -> shared_subscriptions_not_supported; +name(16#9F) -> connection_rate_exceeded; +name(16#A0) -> maximum_connect_time; +name(16#A1) -> subscription_identifiers_not_supported; +name(16#A2) -> wildcard_subscriptions_not_supported; +name(_Code) -> unknown_error. + +text(16#00) -> <<"Success">>; +text(16#01) -> <<"Granted QoS 1">>; +text(16#02) -> <<"Granted QoS 2">>; +text(16#04) -> <<"Disconnect with Will Message">>; +text(16#10) -> <<"No matching subscribers">>; +text(16#11) -> <<"No subscription existed">>; +text(16#18) -> <<"Continue authentication">>; +text(16#19) -> <<"Re-authenticate">>; +text(16#80) -> <<"Unspecified error">>; +text(16#81) -> <<"Malformed Packet">>; +text(16#82) -> <<"Protocol Error">>; +text(16#83) -> <<"Implementation specific error">>; +text(16#84) -> <<"Unsupported Protocol Version">>; +text(16#85) -> <<"Client Identifier not valid">>; +text(16#86) -> <<"Bad User Name or Password">>; +text(16#87) -> <<"Not authorized">>; +text(16#88) -> <<"Server unavailable">>; +text(16#89) -> <<"Server busy">>; +text(16#8A) -> <<"Banned">>; +text(16#8B) -> <<"Server shutting down">>; +text(16#8C) -> <<"Bad authentication method">>; +text(16#8D) -> <<"Keep Alive timeout">>; +text(16#8E) -> <<"Session taken over">>; +text(16#8F) -> <<"Topic Filter invalid">>; +text(16#90) -> <<"Topic Name invalid">>; +text(16#91) -> <<"Packet Identifier in use">>; +text(16#92) -> <<"Packet Identifier not found">>; +text(16#93) -> <<"Receive Maximum exceeded">>; +text(16#94) -> <<"Topic Alias invalid">>; +text(16#95) -> <<"Packet too large">>; +text(16#96) -> <<"Message rate too high">>; +text(16#97) -> <<"Quota exceeded">>; +text(16#98) -> <<"Administrative action">>; +text(16#99) -> <<"Payload format invalid">>; +text(16#9A) -> <<"Retain not supported">>; +text(16#9B) -> <<"QoS not supported">>; +text(16#9C) -> <<"Use another server">>; +text(16#9D) -> <<"Server moved">>; +text(16#9E) -> <<"Shared Subscriptions not supported">>; +text(16#9F) -> <<"Connection rate exceeded">>; +text(16#A0) -> <<"Maximum connect time">>; +text(16#A1) -> <<"Subscription Identifiers not supported">>; +text(16#A2) -> <<"Wildcard Subscriptions not supported">>; +text(_Code) -> <<"Unknown error">>. + +compat(connack, 16#80) -> ?CONNACK_PROTO_VER; +compat(connack, 16#81) -> ?CONNACK_PROTO_VER; +compat(connack, 16#82) -> ?CONNACK_PROTO_VER; +compat(connack, 16#83) -> ?CONNACK_PROTO_VER; +compat(connack, 16#84) -> ?CONNACK_PROTO_VER; +compat(connack, 16#85) -> ?CONNACK_INVALID_ID; +compat(connack, 16#86) -> ?CONNACK_CREDENTIALS; +compat(connack, 16#87) -> ?CONNACK_AUTH; +compat(connack, 16#88) -> ?CONNACK_SERVER; +compat(connack, 16#89) -> ?CONNACK_SERVER; +compat(connack, 16#8A) -> ?CONNACK_AUTH; +compat(connack, 16#8B) -> ?CONNACK_SERVER; +compat(connack, 16#8C) -> ?CONNACK_AUTH; +compat(connack, 16#90) -> ?CONNACK_SERVER; +compat(connack, 16#97) -> ?CONNACK_SERVER; +compat(connack, 16#9C) -> ?CONNACK_SERVER; +compat(connack, 16#9D) -> ?CONNACK_SERVER; +compat(connack, 16#9F) -> ?CONNACK_SERVER; + +compat(suback, Code) when Code =< ?QOS_2 -> Code; +compat(suback, Code) when Code >= 16#80 -> 16#80; + +compat(unsuback, _Code) -> undefined. diff --git a/src/emqx_router.erl b/src/emqx_router.erl new file mode 100644 index 000000000..0463526d6 --- /dev/null +++ b/src/emqx_router.erl @@ -0,0 +1,221 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_router). + +-behaviour(gen_server). + +-include("emqx.hrl"). +-include_lib("ekka/include/ekka.hrl"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-export([start_link/2]). + +%% Route APIs +-export([add_route/1, add_route/2]). +-export([do_add_route/1, do_add_route/2]). +-export([match_routes/1, lookup_routes/1, has_routes/1]). +-export([delete_route/1, delete_route/2]). +-export([do_delete_route/1, do_delete_route/2]). +-export([print_routes/1]). +-export([topics/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-type(group() :: binary()). +-type(destination() :: node() | {group(), node()}). + +-define(ROUTE, emqx_route). + +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?ROUTE, [ + {type, bag}, + {ram_copies, [node()]}, + {record_name, route}, + {attributes, record_info(fields, route)}, + {storage_properties, [{ets, [{read_concurrency, true}, + {write_concurrency, true}]}]}]); +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?ROUTE). + +%%------------------------------------------------------------------------------ +%% Start a router +%%------------------------------------------------------------------------------ + +-spec(start_link(atom(), pos_integer()) -> emqx_types:startlink_ret()). +start_link(Pool, Id) -> + gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, + ?MODULE, [Pool, Id], [{hibernate_after, 1000}]). + +%%------------------------------------------------------------------------------ +%% Route APIs +%%------------------------------------------------------------------------------ + +-spec(add_route(emqx_topic:topic()) -> ok | {error, term()}). +add_route(Topic) when is_binary(Topic) -> + add_route(Topic, node()). + +-spec(add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +add_route(Topic, Dest) when is_binary(Topic) -> + call(pick(Topic), {add_route, Topic, Dest}). + +-spec(do_add_route(emqx_topic:topic()) -> ok | {error, term()}). +do_add_route(Topic) when is_binary(Topic) -> + do_add_route(Topic, node()). + +-spec(do_add_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +do_add_route(Topic, Dest) when is_binary(Topic) -> + Route = #route{topic = Topic, dest = Dest}, + case lists:member(Route, lookup_routes(Topic)) of + true -> ok; + false -> + ok = emqx_router_helper:monitor(Dest), + case emqx_topic:wildcard(Topic) of + true -> trans(fun insert_trie_route/1, [Route]); + false -> insert_direct_route(Route) + end + end. + +%% @doc Match routes +-spec(match_routes(emqx_topic:topic()) -> [emqx_types:route()]). +match_routes(Topic) when is_binary(Topic) -> + %% Optimize: routing table will be replicated to all router nodes. + Matched = mnesia:ets(fun emqx_trie:match/1, [Topic]), + lists:append([lookup_routes(To) || To <- [Topic | Matched]]). + +-spec(lookup_routes(emqx_topic:topic()) -> [emqx_types:route()]). +lookup_routes(Topic) -> + ets:lookup(?ROUTE, Topic). + +-spec(has_routes(emqx_topic:topic()) -> boolean()). +has_routes(Topic) when is_binary(Topic) -> + ets:member(?ROUTE, Topic). + +-spec(delete_route(emqx_topic:topic()) -> ok | {error, term()}). +delete_route(Topic) when is_binary(Topic) -> + delete_route(Topic, node()). + +-spec(delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +delete_route(Topic, Dest) when is_binary(Topic) -> + call(pick(Topic), {delete_route, Topic, Dest}). + +-spec(do_delete_route(emqx_topic:topic()) -> ok | {error, term()}). +do_delete_route(Topic) when is_binary(Topic) -> + do_delete_route(Topic, node()). + +-spec(do_delete_route(emqx_topic:topic(), destination()) -> ok | {error, term()}). +do_delete_route(Topic, Dest) -> + Route = #route{topic = Topic, dest = Dest}, + case emqx_topic:wildcard(Topic) of + true -> trans(fun delete_trie_route/1, [Route]); + false -> delete_direct_route(Route) + end. + +-spec(topics() -> list(emqx_topic:topic())). +topics() -> + mnesia:dirty_all_keys(?ROUTE). + +%% @doc Print routes to a topic +-spec(print_routes(emqx_topic:topic()) -> ok). +print_routes(Topic) -> + lists:foreach(fun(#route{topic = To, dest = Dest}) -> + io:format("~s -> ~s~n", [To, Dest]) + end, match_routes(Topic)). + +call(Router, Msg) -> + gen_server:call(Router, Msg, infinity). + +pick(Topic) -> + gproc_pool:pick_worker(router_pool, Topic). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Pool, Id]) -> + true = gproc_pool:connect_worker(Pool, {Pool, Id}), + {ok, #{pool => Pool, id => Id}}. + +handle_call({add_route, Topic, Dest}, _From, State) -> + Ok = do_add_route(Topic, Dest), + {reply, Ok, State}; + +handle_call({delete_route, Topic, Dest}, _From, State) -> + Ok = do_delete_route(Topic, Dest), + {reply, Ok, State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[Router] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[Router] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + emqx_logger:error("[Router] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #{pool := Pool, id := Id}) -> + gproc_pool:disconnect_worker(Pool, {Pool, Id}). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +insert_direct_route(Route) -> + mnesia:async_dirty(fun mnesia:write/3, [?ROUTE, Route, sticky_write]). + +insert_trie_route(Route = #route{topic = Topic}) -> + case mnesia:wread({?ROUTE, Topic}) of + [] -> emqx_trie:insert(Topic); + _ -> ok + end, + mnesia:write(?ROUTE, Route, sticky_write). + +delete_direct_route(Route) -> + mnesia:async_dirty(fun mnesia:delete_object/3, [?ROUTE, Route, sticky_write]). + +delete_trie_route(Route = #route{topic = Topic}) -> + case mnesia:wread({?ROUTE, Topic}) of + [Route] -> %% Remove route and trie + ok = mnesia:delete_object(?ROUTE, Route, sticky_write), + emqx_trie:delete(Topic); + [_|_] -> %% Remove route only + mnesia:delete_object(?ROUTE, Route, sticky_write); + [] -> ok + end. + +%% @private +-spec(trans(function(), list(any())) -> ok | {error, term()}). +trans(Fun, Args) -> + case mnesia:transaction(Fun, Args) of + {atomic, Ok} -> Ok; + {aborted, Reason} -> {error, Reason} + end. + diff --git a/src/emqx_router_helper.erl b/src/emqx_router_helper.erl new file mode 100644 index 000000000..c32800a24 --- /dev/null +++ b/src/emqx_router_helper.erl @@ -0,0 +1,163 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_router_helper). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +%% API +-export([start_link/0, monitor/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +%% Internal export +-export([stats_fun/0]). + +-record(routing_node, {name, const = unused}). + +-define(ROUTE, emqx_route). +-define(ROUTING_NODE, emqx_routing_node). +-define(LOCK, {?MODULE, cleanup_routes}). + +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?ROUTING_NODE, [ + {type, set}, + {ram_copies, [node()]}, + {record_name, routing_node}, + {attributes, record_info(fields, routing_node)}, + {storage_properties, [{ets, [{read_concurrency, true}]}]}]); + +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?ROUTING_NODE). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% @doc Starts the router helper +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%% @doc Monitor routing node +-spec(monitor(node() | {binary(), node()}) -> ok). +monitor({_Group, Node}) -> + monitor(Node); +monitor(Node) when is_atom(Node) -> + case ekka:is_member(Node) + orelse ets:member(?ROUTING_NODE, Node) of + true -> ok; + false -> mnesia:dirty_write(?ROUTING_NODE, #routing_node{name = Node}) + end. + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + ok = ekka:monitor(membership), + {ok, _} = mnesia:subscribe({table, ?ROUTING_NODE, simple}), + Nodes = lists:foldl( + fun(Node, Acc) -> + case ekka:is_member(Node) of + true -> Acc; + false -> true = erlang:monitor_node(Node, true), + [Node | Acc] + end + end, [], mnesia:dirty_all_keys(?ROUTING_NODE)), + ok = emqx_stats:update_interval(route_stats, fun ?MODULE:stats_fun/0), + {ok, #{nodes => Nodes}, hibernate}. + +handle_call(Req, _From, State) -> + emqx_logger:error("[RouterHelper] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[RouterHelper] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({mnesia_table_event, {write, {?ROUTING_NODE, Node, _}, _}}, State = #{nodes := Nodes}) -> + case ekka:is_member(Node) orelse lists:member(Node, Nodes) of + true -> {noreply, State}; + false -> + true = erlang:monitor_node(Node, true), + {noreply, State#{nodes := [Node | Nodes]}} + end; + +handle_info({mnesia_table_event, {delete, {?ROUTING_NODE, _Node}, _}}, State) -> + %% ignore + {noreply, State}; + +handle_info({mnesia_table_event, Event}, State) -> + emqx_logger:error("[RouterHelper] unexpected mnesia_table_event: ~p", [Event]), + {noreply, State}; + +handle_info({nodedown, Node}, State = #{nodes := Nodes}) -> + global:trans({?LOCK, self()}, + fun() -> + mnesia:transaction(fun cleanup_routes/1, [Node]) + end), + ok = mnesia:dirty_delete(?ROUTING_NODE, Node), + {noreply, State#{nodes := lists:delete(Node, Nodes)}, hibernate}; + +handle_info({membership, {mnesia, down, Node}}, State) -> + handle_info({nodedown, Node}, State); + +handle_info({membership, _Event}, State) -> + {noreply, State}; + +handle_info(Info, State) -> + emqx_logger:error("[RouteHelper] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok = ekka:unmonitor(membership), + emqx_stats:cancel_update(route_stats), + mnesia:unsubscribe({table, ?ROUTING_NODE, simple}). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +stats_fun() -> + case ets:info(?ROUTE, size) of + undefined -> ok; + Size -> + emqx_stats:setstat('routes/count', 'routes/max', Size), + emqx_stats:setstat('topics/count', 'topics/max', Size) + end. + +cleanup_routes(Node) -> + Patterns = [#route{_ = '_', dest = Node}, + #route{_ = '_', dest = {'_', Node}}], + [mnesia:delete_object(?ROUTE, Route, write) + || Pat <- Patterns, Route <- mnesia:match_object(?ROUTE, Pat, write)]. + diff --git a/src/emqttd_trace_sup.erl b/src/emqx_router_sup.erl similarity index 55% rename from src/emqttd_trace_sup.erl rename to src/emqx_router_sup.erl index 35264e017..945d7910f 100644 --- a/src/emqttd_trace_sup.erl +++ b/src/emqx_router_sup.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,25 +11,29 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_trace_sup). - --author("Feng Lee "). +-module(emqx_router_sup). -behaviour(supervisor). -%% API -export([start_link/0]). -%% Supervisor callbacks -export([init/1]). start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - Trace = {trace, {emqttd_trace, start_link, []}, - permanent, 5000, worker, [emqttd_trace]}, - {ok, {{one_for_one, 10, 3600}, [Trace]}}. + %% Router helper + Helper = #{id => helper, + start => {emqx_router_helper, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_router_helper]}, + + %% Router pool + RouterPool = emqx_pool_sup:spec([router_pool, hash, + {emqx_router, start_link, []}]), + {ok, {{one_for_all, 0, 1}, [Helper, RouterPool]}}. diff --git a/src/emqx_rpc.erl b/src/emqx_rpc.erl new file mode 100644 index 000000000..d4433bf6c --- /dev/null +++ b/src/emqx_rpc.erl @@ -0,0 +1,31 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc wrap gen_rpc? +-module(emqx_rpc). + +-export([call/4, cast/4]). +-export([multicall/4]). + +-define(RPC, gen_rpc). + +call(Node, Mod, Fun, Args) -> + ?RPC:call(Node, Mod, Fun, Args). + +multicall(Nodes, Mod, Fun, Args) -> + ?RPC:multicall(Nodes, Mod, Fun, Args). + +cast(Node, Mod, Fun, Args) -> + ?RPC:cast(Node, Mod, Fun, Args). + diff --git a/src/emqx_sequence.erl b/src/emqx_sequence.erl new file mode 100644 index 000000000..33bb5edda --- /dev/null +++ b/src/emqx_sequence.erl @@ -0,0 +1,60 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sequence). + +-export([create/1, nextval/2, currval/2, reclaim/2, delete/1]). + +-type(key() :: term()). +-type(name() :: atom()). +-type(seqid() :: non_neg_integer()). + +-export_type([seqid/0]). + +%% @doc Create a sequence. +-spec(create(name()) -> ok). +create(Name) -> + emqx_tables:new(Name, [public, set, {write_concurrency, true}]). + +%% @doc Next value of the sequence. +-spec(nextval(name(), key()) -> seqid()). +nextval(Name, Key) -> + ets:update_counter(Name, Key, {2, 1}, {Key, 0}). + +%% @doc Current value of the sequence. +-spec(currval(name(), key()) -> seqid()). +currval(Name, Key) -> + try ets:lookup_element(Name, Key, 2) + catch + error:badarg -> 0 + end. + +%% @doc Reclaim a sequence id. +-spec(reclaim(name(), key()) -> seqid()). +reclaim(Name, Key) -> + try ets:update_counter(Name, Key, {2, -1, 0, 0}) of + 0 -> ets:delete_object(Name, {Key, 0}), 0; + I -> I + catch + error:badarg -> 0 + end. + +%% @doc Delete the sequence. +-spec(delete(name()) -> boolean()). +delete(Name) -> + case ets:info(Name, name) of + Name -> ets:delete(Name); + undefined -> false + end. + diff --git a/src/emqx_session.erl b/src/emqx_session.erl new file mode 100644 index 000000000..5d5381f6e --- /dev/null +++ b/src/emqx_session.erl @@ -0,0 +1,1019 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @doc +%% A stateful interaction between a Client and a Server. Some Sessions +%% last only as long as the Network Connection, others can span multiple +%% consecutive Network Connections between a Client and a Server. +%% +%% The Session State in the Server consists of: +%% +%% The existence of a Session, even if the rest of the Session State is empty. +%% +%% The Clients subscriptions, including any Subscription Identifiers. +%% +%% QoS 1 and QoS 2 messages which have been sent to the Client, but have not +%% been completely acknowledged. +%% +%% QoS 1 and QoS 2 messages pending transmission to the Client and OPTIONALLY +%% QoS 0 messages pending transmission to the Client. +%% +%% QoS 2 messages which have been received from the Client, but have not been +%% completely acknowledged.The Will Message and the Will Delay Interval +%% +%% If the Session is currently not connected, the time at which the Session +%% will end and Session State will be discarded. +%% @end + +-module(emqx_session). + +-behaviour(gen_server). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-export([start_link/1]). +-export([info/1, attrs/1]). +-export([stats/1]). +-export([resume/2, discard/2]). +-export([update_expiry_interval/2]). +-export([subscribe/2, subscribe/4]). +-export([publish/3]). +-export([puback/2, puback/3]). +-export([pubrec/2, pubrec/3]). +-export([pubrel/3, pubcomp/3]). +-export([unsubscribe/2, unsubscribe/4]). +-export([close/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-import(emqx_zone, [get_env/2, get_env/3]). + +-record(state, { + %% Idle timeout + idle_timeout :: pos_integer(), + + %% Clean Start Flag + clean_start = false :: boolean(), + + %% Client Binding: local | remote + binding = local :: local | remote, + + %% ClientId: Identifier of Session + client_id :: binary(), + + %% Username + username :: binary() | undefined, + + %% Connection pid binding with session + conn_pid :: pid(), + + %% Old Connection Pid that has been kickout + old_conn_pid :: pid(), + + %% Next packet id of the session + next_pkt_id = 1 :: emqx_mqtt_types:packet_id(), + + %% Max subscriptions + max_subscriptions :: non_neg_integer(), + + %% Client’s Subscriptions. + subscriptions :: map(), + + %% Upgrade QoS? + upgrade_qos = false :: boolean(), + + %% Client <- Broker: Inflight QoS1, QoS2 messages sent to the client but unacked. + inflight :: emqx_inflight:inflight(), + + %% Max Inflight Size. DEPRECATED: Get from inflight + %% max_inflight = 32 :: non_neg_integer(), + + %% Retry interval for redelivering QoS1/2 messages + retry_interval = 20000 :: timeout(), + + %% Retry Timer + retry_timer :: reference() | undefined, + + %% All QoS1, QoS2 messages published to when client is disconnected. + %% QoS 1 and QoS 2 messages pending transmission to the Client. + %% + %% Optionally, QoS 0 messages pending transmission to the Client. + mqueue :: emqx_mqueue:mqueue(), + + %% Client -> Broker: Inflight QoS2 messages received from client and waiting for pubrel. + awaiting_rel :: map(), + + %% Max Packets Awaiting PUBREL + max_awaiting_rel = 100 :: non_neg_integer(), + + %% Awaiting PUBREL Timeout + await_rel_timeout = 20000 :: timeout(), + + %% Awaiting PUBREL Timer + await_rel_timer :: reference() | undefined, + + %% Session Expiry Interval + expiry_interval = 7200 :: timeout(), + + %% Expired Timer + expiry_timer :: reference() | undefined, + + %% Enable Stats + enable_stats :: boolean(), + + %% Stats timer + stats_timer :: reference() | undefined, + + %% GC State + gc_state, + + %% Created at + created_at :: erlang:timestamp(), + + will_msg :: emqx:message(), + + will_delay_timer :: reference() | undefined + + }). + +-type(spid() :: pid()). +-type(attr() :: {atom(), term()}). + +-export_type([attr/0]). + +-define(LOG(Level, Format, Args, _State), + emqx_logger:Level("[Session] " ++ Format, Args)). + +%% @doc Start a session proc. +-spec(start_link(SessAttrs :: map()) -> {ok, pid()}). +start_link(SessAttrs) -> + proc_lib:start_link(?MODULE, init, [[self(), SessAttrs]]). + +%% @doc Get session info +-spec(info(spid() | #state{}) -> list({atom(), term()})). +info(SPid) when is_pid(SPid) -> + gen_server:call(SPid, info, infinity); + +info(State = #state{conn_pid = ConnPid, + next_pkt_id = PktId, + max_subscriptions = MaxSubscriptions, + subscriptions = Subscriptions, + upgrade_qos = UpgradeQoS, + inflight = Inflight, + retry_interval = RetryInterval, + mqueue = MQueue, + awaiting_rel = AwaitingRel, + max_awaiting_rel = MaxAwaitingRel, + await_rel_timeout = AwaitRelTimeout}) -> + attrs(State) ++ [{conn_pid, ConnPid}, + {next_pkt_id, PktId}, + {max_subscriptions, MaxSubscriptions}, + {subscriptions, Subscriptions}, + {upgrade_qos, UpgradeQoS}, + {inflight, Inflight}, + {retry_interval, RetryInterval}, + {mqueue_len, MQueue}, + {awaiting_rel, AwaitingRel}, + {max_awaiting_rel, MaxAwaitingRel}, + {await_rel_timeout, AwaitRelTimeout}]. + +%% @doc Get session attrs +-spec(attrs(spid() | #state{}) -> list({atom(), term()})). +attrs(SPid) when is_pid(SPid) -> + gen_server:call(SPid, attrs, infinity); + +attrs(#state{clean_start = CleanStart, + binding = Binding, + client_id = ClientId, + username = Username, + expiry_interval = ExpiryInterval, + created_at = CreatedAt}) -> + [{clean_start, CleanStart}, + {binding, Binding}, + {client_id, ClientId}, + {username, Username}, + {expiry_interval, ExpiryInterval div 1000}, + {created_at, CreatedAt}]. + +-spec(stats(spid() | #state{}) -> list({atom(), non_neg_integer()})). +stats(SPid) when is_pid(SPid) -> + gen_server:call(SPid, stats, infinity); + +stats(#state{max_subscriptions = MaxSubscriptions, + subscriptions = Subscriptions, + inflight = Inflight, + mqueue = MQueue, + max_awaiting_rel = MaxAwaitingRel, + awaiting_rel = AwaitingRel}) -> + lists:append(emqx_misc:proc_stats(), + [{max_subscriptions, MaxSubscriptions}, + {subscriptions_count, maps:size(Subscriptions)}, + {max_inflight, emqx_inflight:max_size(Inflight)}, + {inflight_len, emqx_inflight:size(Inflight)}, + {max_mqueue, emqx_mqueue:max_len(MQueue)}, + {mqueue_len, emqx_mqueue:len(MQueue)}, + {mqueue_dropped, emqx_mqueue:dropped(MQueue)}, + {max_awaiting_rel, MaxAwaitingRel}, + {awaiting_rel_len, maps:size(AwaitingRel)}, + {deliver_msg, emqx_pd:get_counter(deliver_stats)}, + {enqueue_msg, emqx_pd:get_counter(enqueue_stats)}]). + +%%------------------------------------------------------------------------------ +%% PubSub API +%%------------------------------------------------------------------------------ + +-spec(subscribe(spid(), list({emqx_topic:topic(), emqx_types:subopts()})) -> ok). +subscribe(SPid, RawTopicFilters) when is_list(RawTopicFilters) -> + TopicFilters = [emqx_topic:parse(RawTopic, maps:merge(?DEFAULT_SUBOPTS, SubOpts)) + || {RawTopic, SubOpts} <- RawTopicFilters], + subscribe(SPid, undefined, #{}, TopicFilters). + +-spec(subscribe(spid(), emqx_mqtt_types:packet_id(), + emqx_mqtt_types:properties(), emqx_mqtt_types:topic_filters()) -> ok). +subscribe(SPid, PacketId, Properties, TopicFilters) -> + SubReq = {PacketId, Properties, TopicFilters}, + gen_server:cast(SPid, {subscribe, self(), SubReq}). + +%% @doc Called by connection processes when publishing messages +-spec(publish(spid(), emqx_mqtt_types:packet_id(), emqx_types:message()) + -> emqx_types:deliver_results() | {error, term()}). +publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_0}) -> + %% Publish QoS0 message directly + emqx_broker:publish(Msg); + +publish(_SPid, _PacketId, Msg = #message{qos = ?QOS_1}) -> + %% Publish QoS1 message directly + emqx_broker:publish(Msg); + +publish(SPid, PacketId, Msg = #message{qos = ?QOS_2, timestamp = Ts}) -> + %% Register QoS2 message packet ID (and timestamp) to session, then publish + case gen_server:call(SPid, {register_publish_packet_id, PacketId, Ts}, infinity) of + ok -> emqx_broker:publish(Msg); + {error, Reason} -> {error, Reason} + end. + +-spec(puback(spid(), emqx_mqtt_types:packet_id()) -> ok). +puback(SPid, PacketId) -> + gen_server:cast(SPid, {puback, PacketId, ?RC_SUCCESS}). + +-spec(puback(spid(), emqx_mqtt_types:packet_id(), emqx_mqtt_types:reason_code()) -> ok). +puback(SPid, PacketId, ReasonCode) -> + gen_server:cast(SPid, {puback, PacketId, ReasonCode}). + +-spec(pubrec(spid(), emqx_mqtt_types:packet_id()) -> ok | {error, emqx_mqtt_types:reason_code()}). +pubrec(SPid, PacketId) -> + pubrec(SPid, PacketId, ?RC_SUCCESS). + +-spec(pubrec(spid(), emqx_mqtt_types:packet_id(), emqx_mqtt_types:reason_code()) + -> ok | {error, emqx_mqtt_types:reason_code()}). +pubrec(SPid, PacketId, ReasonCode) -> + gen_server:call(SPid, {pubrec, PacketId, ReasonCode}, infinity). + +-spec(pubrel(spid(), emqx_mqtt_types:packet_id(), emqx_mqtt_types:reason_code()) + -> ok | {error, emqx_mqtt_types:reason_code()}). +pubrel(SPid, PacketId, ReasonCode) -> + gen_server:call(SPid, {pubrel, PacketId, ReasonCode}, infinity). + +-spec(pubcomp(spid(), emqx_mqtt_types:packet_id(), emqx_mqtt_types:reason_code()) -> ok). +pubcomp(SPid, PacketId, ReasonCode) -> + gen_server:cast(SPid, {pubcomp, PacketId, ReasonCode}). + +-spec(unsubscribe(spid(), emqx_types:topic_table()) -> ok). +unsubscribe(SPid, RawTopicFilters) when is_list(RawTopicFilters) -> + TopicFilters = lists:map(fun({RawTopic, Opts}) -> + emqx_topic:parse(RawTopic, Opts); + (RawTopic) when is_binary(RawTopic) -> + emqx_topic:parse(RawTopic) + end, RawTopicFilters), + unsubscribe(SPid, undefined, #{}, TopicFilters). + +-spec(unsubscribe(spid(), emqx_mqtt_types:packet_id(), + emqx_mqtt_types:properties(), emqx_mqtt_types:topic_filters()) -> ok). +unsubscribe(SPid, PacketId, Properties, TopicFilters) -> + UnsubReq = {PacketId, Properties, TopicFilters}, + gen_server:cast(SPid, {unsubscribe, self(), UnsubReq}). + +-spec(resume(spid(), map()) -> ok). +resume(SPid, SessAttrs) -> + gen_server:cast(SPid, {resume, SessAttrs}). + +%% @doc Discard the session +-spec(discard(spid(), ByPid :: pid()) -> ok). +discard(SPid, ByPid) -> + gen_server:call(SPid, {discard, ByPid}, infinity). + +-spec(update_expiry_interval(spid(), timeout()) -> ok). +update_expiry_interval(SPid, Interval) -> + gen_server:cast(SPid, {update_expiry_interval, Interval}). + +-spec(close(spid()) -> ok). +close(SPid) -> + gen_server:call(SPid, close, infinity). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Parent, #{zone := Zone, + client_id := ClientId, + username := Username, + conn_pid := ConnPid, + clean_start := CleanStart, + expiry_interval := ExpiryInterval, + max_inflight := MaxInflight, + will_msg := WillMsg}]) -> + process_flag(trap_exit, true), + true = link(ConnPid), + emqx_logger:set_metadata_client_id(ClientId), + GcPolicy = emqx_zone:get_env(Zone, force_gc_policy, false), + IdleTimout = get_env(Zone, idle_timeout, 30000), + State = #state{idle_timeout = IdleTimout, + clean_start = CleanStart, + binding = binding(ConnPid), + client_id = ClientId, + username = Username, + conn_pid = ConnPid, + subscriptions = #{}, + max_subscriptions = get_env(Zone, max_subscriptions, 0), + upgrade_qos = get_env(Zone, upgrade_qos, false), + inflight = emqx_inflight:new(MaxInflight), + mqueue = init_mqueue(Zone), + retry_interval = get_env(Zone, retry_interval, 0), + awaiting_rel = #{}, + await_rel_timeout = get_env(Zone, await_rel_timeout), + max_awaiting_rel = get_env(Zone, max_awaiting_rel), + expiry_interval = ExpiryInterval, + enable_stats = get_env(Zone, enable_stats, true), + gc_state = emqx_gc:init(GcPolicy), + created_at = os:timestamp(), + will_msg = WillMsg + }, + ok = emqx_sm:register_session(ClientId, self()), + true = emqx_sm:set_session_attrs(ClientId, attrs(State)), + true = emqx_sm:set_session_stats(ClientId, stats(State)), + emqx_hooks:run('session.created', [#{client_id => ClientId}, info(State)]), + ok = emqx_misc:init_proc_mng_policy(Zone), + ok = proc_lib:init_ack(Parent, {ok, self()}), + gen_server:enter_loop(?MODULE, [{hibernate_after, IdleTimout}], State). + +init_mqueue(Zone) -> + emqx_mqueue:init(#{max_len => get_env(Zone, max_mqueue_len, 1000), + store_qos0 => get_env(Zone, mqueue_store_qos0, true), + priorities => get_env(Zone, mqueue_priorities), + default_priority => get_env(Zone, mqueue_default_priority) + }). + +binding(ConnPid) -> + case node(ConnPid) =:= node() of true -> local; false -> remote end. + +handle_call(info, _From, State) -> + reply(info(State), State); + +handle_call(attrs, _From, State) -> + reply(attrs(State), State); + +handle_call(stats, _From, State) -> + reply(stats(State), State); + +handle_call({discard, ByPid}, _From, State = #state{conn_pid = undefined}) -> + ?LOG(warning, "Discarded by ~p", [ByPid], State), + {stop, {shutdown, discarded}, ok, State}; + +handle_call({discard, ByPid}, _From, State = #state{client_id = ClientId, conn_pid = ConnPid}) -> + ?LOG(warning, "Conn ~p is discarded by ~p", [ConnPid, ByPid], State), + ConnPid ! {shutdown, discard, {ClientId, ByPid}}, + {stop, {shutdown, discarded}, ok, State}; + +%% PUBLISH: This is only to register packetId to session state. +%% The actual message dispatching should be done by the caller (e.g. connection) process. +handle_call({register_publish_packet_id, PacketId, Ts}, _From, + State = #state{awaiting_rel = AwaitingRel}) -> + reply( + case is_awaiting_full(State) of + false -> + case maps:is_key(PacketId, AwaitingRel) of + true -> + {{error, ?RC_PACKET_IDENTIFIER_IN_USE}, State}; + false -> + State1 = State#state{awaiting_rel = maps:put(PacketId, Ts, AwaitingRel)}, + {ok, ensure_stats_timer(ensure_await_rel_timer(State1))} + end; + true -> + ?LOG(warning, "Dropped qos2 packet ~w for too many awaiting_rel", [PacketId], State), + emqx_metrics:trans(inc, 'messages/qos2/dropped'), + {{error, ?RC_RECEIVE_MAXIMUM_EXCEEDED}, State} + end); + +%% PUBREC: +handle_call({pubrec, PacketId, _ReasonCode}, _From, State = #state{inflight = Inflight}) -> + reply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + {ok, ensure_stats_timer(acked(pubrec, PacketId, State))}; + false -> + ?LOG(warning, "The PUBREC PacketId ~w is not found.", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubrec/missed'), + {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} + end); + +%% PUBREL: +handle_call({pubrel, PacketId, _ReasonCode}, _From, State = #state{awaiting_rel = AwaitingRel}) -> + reply( + case maps:take(PacketId, AwaitingRel) of + {_Ts, AwaitingRel1} -> + {ok, ensure_stats_timer(State#state{awaiting_rel = AwaitingRel1})}; + error -> + ?LOG(warning, "The PUBREL PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubrel/missed'), + {{error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}, State} + end); + +handle_call(close, _From, State) -> + {stop, normal, ok, State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[Session] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +%% SUBSCRIBE: +handle_cast({subscribe, FromPid, {PacketId, _Properties, TopicFilters}}, + State = #state{client_id = ClientId, subscriptions = Subscriptions}) -> + {ReasonCodes, Subscriptions1} = + lists:foldr(fun({Topic, SubOpts = #{qos := QoS}}, {RcAcc, SubMap}) -> + {[QoS|RcAcc], case maps:find(Topic, SubMap) of + {ok, SubOpts} -> + emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]), + SubMap; + {ok, _SubOpts} -> + emqx_broker:set_subopts(Topic, SubOpts), + %% Why??? + emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => false}]), + maps:put(Topic, SubOpts, SubMap); + error -> + emqx_broker:subscribe(Topic, ClientId, SubOpts), + emqx_hooks:run('session.subscribed', [#{client_id => ClientId}, Topic, SubOpts#{first => true}]), + maps:put(Topic, SubOpts, SubMap) + end} + end, {[], Subscriptions}, TopicFilters), + suback(FromPid, PacketId, ReasonCodes), + noreply(ensure_stats_timer(State#state{subscriptions = Subscriptions1})); + +%% UNSUBSCRIBE: +handle_cast({unsubscribe, From, {PacketId, _Properties, TopicFilters}}, + State = #state{client_id = ClientId, subscriptions = Subscriptions}) -> + {ReasonCodes, Subscriptions1} = + lists:foldr(fun({Topic, _SubOpts}, {Acc, SubMap}) -> + case maps:find(Topic, SubMap) of + {ok, SubOpts} -> + ok = emqx_broker:unsubscribe(Topic), + emqx_hooks:run('session.unsubscribed', [#{client_id => ClientId}, Topic, SubOpts]), + {[?RC_SUCCESS|Acc], maps:remove(Topic, SubMap)}; + error -> + {[?RC_NO_SUBSCRIPTION_EXISTED|Acc], SubMap} + end + end, {[], Subscriptions}, TopicFilters), + unsuback(From, PacketId, ReasonCodes), + noreply(ensure_stats_timer(State#state{subscriptions = Subscriptions1})); + +%% PUBACK: +handle_cast({puback, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) -> + noreply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + ensure_stats_timer(dequeue(acked(puback, PacketId, State))); + false -> + ?LOG(warning, "The PUBACK PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/puback/missed'), + State + end); + +%% PUBCOMP: +handle_cast({pubcomp, PacketId, _ReasonCode}, State = #state{inflight = Inflight}) -> + noreply( + case emqx_inflight:contain(PacketId, Inflight) of + true -> + ensure_stats_timer(dequeue(acked(pubcomp, PacketId, State))); + false -> + ?LOG(warning, "The PUBCOMP PacketId ~w is not found", [PacketId], State), + emqx_metrics:trans(inc, 'packets/pubcomp/missed'), + State + end); + +%% RESUME: +handle_cast({resume, #{conn_pid := ConnPid, + will_msg := WillMsg, + expiry_interval := ExpiryInterval, + max_inflight := MaxInflight}}, + State = #state{client_id = ClientId, + conn_pid = OldConnPid, + clean_start = CleanStart, + retry_timer = RetryTimer, + await_rel_timer = AwaitTimer, + expiry_timer = ExpireTimer, + will_delay_timer = WillDelayTimer}) -> + + ?LOG(info, "Resumed by connection ~p ", [ConnPid], State), + + %% Cancel Timers + lists:foreach(fun emqx_misc:cancel_timer/1, + [RetryTimer, AwaitTimer, ExpireTimer, WillDelayTimer]), + + case kick(ClientId, OldConnPid, ConnPid) of + ok -> ?LOG(warning, "Connection ~p kickout ~p", [ConnPid, OldConnPid], State); + ignore -> ok + end, + + true = link(ConnPid), + + State1 = State#state{conn_pid = ConnPid, + binding = binding(ConnPid), + old_conn_pid = OldConnPid, + clean_start = false, + retry_timer = undefined, + awaiting_rel = #{}, + await_rel_timer = undefined, + expiry_timer = undefined, + expiry_interval = ExpiryInterval, + inflight = emqx_inflight:update_size(MaxInflight, State#state.inflight), + will_delay_timer = undefined, + will_msg = WillMsg}, + + %% Clean Session: true -> false??? + CleanStart andalso emqx_sm:set_session_attrs(ClientId, attrs(State1)), + + emqx_hooks:run('session.resumed', [#{client_id => ClientId}, attrs(State)]), + + %% Replay delivery and Dequeue pending messages + noreply(ensure_stats_timer(dequeue(retry_delivery(true, State1)))); + +handle_cast({update_expiry_interval, Interval}, State) -> + {noreply, State#state{expiry_interval = Interval}}; + +handle_cast(Msg, State) -> + emqx_logger:error("[Session] unexpected cast: ~p", [Msg]), + {noreply, State}. + +%% Batch dispatch +handle_info({dispatch, Topic, Msgs}, State) when is_list(Msgs) -> + noreply(lists:foldl( + fun(Msg, St) -> + element(2, handle_info({dispatch, Topic, Msg}, St)) + end, State, Msgs)); + +%% Dispatch message +handle_info({dispatch, Topic, Msg = #message{}}, State) -> + case emqx_shared_sub:is_ack_required(Msg) andalso not has_connection(State) of + true -> + %% Require ack, but we do not have connection + %% negative ack the message so it can try the next subscriber in the group + ok = emqx_shared_sub:nack_no_connection(Msg), + {noreply, State}; + false -> + NewState = handle_dispatch(Topic, Msg, State), + noreply(ensure_stats_timer(maybe_gc({1, msg_size(Msg)}, NewState))) + end; + +%% Do nothing if the client has been disconnected. +handle_info({timeout, Timer, retry_delivery}, State = #state{conn_pid = undefined, retry_timer = Timer}) -> + noreply(State#state{retry_timer = undefined}); + +handle_info({timeout, Timer, retry_delivery}, State = #state{retry_timer = Timer}) -> + noreply(retry_delivery(false, State#state{retry_timer = undefined})); + +handle_info({timeout, Timer, check_awaiting_rel}, State = #state{await_rel_timer = Timer}) -> + State1 = State#state{await_rel_timer = undefined}, + noreply(ensure_stats_timer(expire_awaiting_rel(State1))); + +handle_info({timeout, Timer, emit_stats}, + State = #state{client_id = ClientId, + stats_timer = Timer, + gc_state = GcState}) -> + emqx_metrics:commit(), + _ = emqx_sm:set_session_stats(ClientId, stats(State)), + NewState = State#state{stats_timer = undefined}, + Limits = erlang:get(force_shutdown_policy), + case emqx_misc:conn_proc_mng_policy(Limits) of + continue -> + {noreply, NewState}; + hibernate -> + %% going to hibernate, reset gc stats + GcState1 = emqx_gc:reset(GcState), + {noreply, NewState#state{gc_state = GcState1}, hibernate}; + {shutdown, Reason} -> + ?LOG(warning, "shutdown due to ~p", [Reason], NewState), + shutdown(Reason, NewState) + end; + +handle_info({timeout, Timer, expired}, State = #state{expiry_timer = Timer}) -> + ?LOG(info, "expired, shutdown now.", [], State), + shutdown(expired, State); + +handle_info({timeout, Timer, will_delay}, State = #state{will_msg = WillMsg, will_delay_timer = Timer}) -> + send_willmsg(WillMsg), + {noreply, State#state{will_msg = undefined}}; + +%% ConnPid is shutting down by the supervisor. +handle_info({'EXIT', ConnPid, Reason}, #state{conn_pid = ConnPid}) + when Reason =:= killed; Reason =:= shutdown -> + exit(Reason); + +handle_info({'EXIT', ConnPid, Reason}, State = #state{will_msg = WillMsg, expiry_interval = 0, conn_pid = ConnPid}) -> + send_willmsg(WillMsg), + {stop, Reason, State#state{will_msg = undefined, conn_pid = undefined}}; + +handle_info({'EXIT', ConnPid, _Reason}, State = #state{conn_pid = ConnPid}) -> + State1 = ensure_will_delay_timer(State), + {noreply, ensure_expire_timer(State1#state{conn_pid = undefined})}; + +handle_info({'EXIT', OldPid, _Reason}, State = #state{old_conn_pid = OldPid}) -> + %% ignore + {noreply, State#state{old_conn_pid = undefined}}; + +handle_info({'EXIT', Pid, Reason}, State = #state{conn_pid = ConnPid}) -> + ?LOG(error, "Unexpected EXIT: conn_pid=~p, exit_pid=~p, reason=~p", + [ConnPid, Pid, Reason], State), + {noreply, State}; + +handle_info(Info, State) -> + emqx_logger:error("[Session] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(Reason, #state{will_msg = WillMsg, + client_id = ClientId, + conn_pid = ConnPid, + old_conn_pid = OldConnPid}) -> + send_willmsg(WillMsg), + [maybe_shutdown(Pid, Reason) || Pid <- [ConnPid, OldConnPid]], + emqx_hooks:run('session.terminated', [#{client_id => ClientId}, Reason]). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +maybe_shutdown(undefined, _Reason) -> + ok; +maybe_shutdown(Pid, normal) -> + Pid ! {shutdown, normal}; +maybe_shutdown(Pid, Reason) -> + exit(Pid, Reason). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +has_connection(#state{conn_pid = Pid}) -> + is_pid(Pid) andalso is_process_alive(Pid). + +handle_dispatch(Topic, Msg, State = #state{subscriptions = SubMap}) -> + case maps:find(Topic, SubMap) of + {ok, #{nl := Nl, qos := QoS, rap := Rap, subid := SubId}} -> + run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}, {subid, SubId}], Msg, State); + {ok, #{nl := Nl, qos := QoS, rap := Rap}} -> + run_dispatch_steps([{nl, Nl}, {qos, QoS}, {rap, Rap}], Msg, State); + error -> + dispatch(emqx_message:unset_flag(dup, Msg), State) + end. + +suback(_From, undefined, _ReasonCodes) -> + ignore; +suback(From, PacketId, ReasonCodes) -> + From ! {deliver, {suback, PacketId, ReasonCodes}}. + +unsuback(_From, undefined, _ReasonCodes) -> + ignore; +unsuback(From, PacketId, ReasonCodes) -> + From ! {deliver, {unsuback, PacketId, ReasonCodes}}. + +%%------------------------------------------------------------------------------ +%% Kickout old connection + +kick(_ClientId, undefined, _ConnPid) -> + ignore; +kick(_ClientId, ConnPid, ConnPid) -> + ignore; +kick(ClientId, OldConnPid, ConnPid) -> + unlink(OldConnPid), + OldConnPid ! {shutdown, conflict, {ClientId, ConnPid}}, + %% Clean noproc + receive {'EXIT', OldConnPid, _} -> ok after 1 -> ok end. + +%%------------------------------------------------------------------------------ +%% Replay or Retry Delivery +%%------------------------------------------------------------------------------ + +%% Redeliver at once if force is true +retry_delivery(Force, State = #state{inflight = Inflight}) -> + case emqx_inflight:is_empty(Inflight) of + true -> State; + false -> + SortFun = fun({_, _, Ts1}, {_, _, Ts2}) -> Ts1 < Ts2 end, + Msgs = lists:sort(SortFun, emqx_inflight:values(Inflight)), + retry_delivery(Force, Msgs, os:timestamp(), State) + end. + +retry_delivery(_Force, [], _Now, State) -> + %% Retry again... + ensure_retry_timer(State); + +retry_delivery(Force, [{Type, Msg0, Ts} | Msgs], Now, + State = #state{inflight = Inflight, retry_interval = Interval}) -> + %% Microseconds -> MilliSeconds + Age = timer:now_diff(Now, Ts) div 1000, + if + Force orelse (Age >= Interval) -> + Inflight1 = case {Type, Msg0} of + {publish, {PacketId, Msg}} -> + case emqx_message:is_expired(Msg) of + true -> + emqx_metrics:trans(inc, 'messages/expired'), + emqx_inflight:delete(PacketId, Inflight); + false -> + redeliver({PacketId, Msg}, State), + emqx_inflight:update(PacketId, {publish, {PacketId, Msg}, Now}, Inflight) + end; + {pubrel, PacketId} -> + redeliver({pubrel, PacketId}, State), + emqx_inflight:update(PacketId, {pubrel, PacketId, Now}, Inflight) + end, + retry_delivery(Force, Msgs, Now, State#state{inflight = Inflight1}); + true -> + ensure_retry_timer(Interval - max(0, Age), State) + end. + +%%------------------------------------------------------------------------------ +%% Send Will Message +%%------------------------------------------------------------------------------ +send_willmsg(undefined) -> + ignore; +send_willmsg(WillMsg) -> + emqx_broker:publish(WillMsg). + +%%------------------------------------------------------------------------------ +%% Expire Awaiting Rel +%%------------------------------------------------------------------------------ + +expire_awaiting_rel(State = #state{awaiting_rel = AwaitingRel}) -> + case maps:size(AwaitingRel) of + 0 -> State; + _ -> expire_awaiting_rel(lists:keysort(2, maps:to_list(AwaitingRel)), os:timestamp(), State) + end. + +expire_awaiting_rel([], _Now, State) -> + State#state{await_rel_timer = undefined}; + +expire_awaiting_rel([{PacketId, Ts} | More], Now, + State = #state{awaiting_rel = AwaitingRel, await_rel_timeout = Timeout}) -> + case (timer:now_diff(Now, Ts) div 1000) of + Age when Age >= Timeout -> + emqx_metrics:trans(inc, 'messages/qos2/expired'), + ?LOG(warning, "Dropped qos2 packet ~s for await_rel_timeout", [PacketId], State), + expire_awaiting_rel(More, Now, State#state{awaiting_rel = maps:remove(PacketId, AwaitingRel)}); + Age -> + ensure_await_rel_timer(Timeout - max(0, Age), State) + end. + +%%------------------------------------------------------------------------------ +%% Check awaiting rel +%%------------------------------------------------------------------------------ + +is_awaiting_full(#state{max_awaiting_rel = 0}) -> + false; +is_awaiting_full(#state{awaiting_rel = AwaitingRel, max_awaiting_rel = MaxLen}) -> + maps:size(AwaitingRel) >= MaxLen. + +%%------------------------------------------------------------------------------ +%% Dispatch Messages +%%------------------------------------------------------------------------------ + +run_dispatch_steps([], Msg, State) -> + dispatch(Msg, State); +run_dispatch_steps([{nl, 1}|_Steps], #message{from = ClientId}, State = #state{client_id = ClientId}) -> + State; +run_dispatch_steps([{nl, _}|Steps], Msg, State) -> + run_dispatch_steps(Steps, Msg, State); +run_dispatch_steps([{qos, SubQoS}|Steps], Msg0 = #message{qos = PubQoS}, State = #state{upgrade_qos = false}) -> + %% Ack immediately if a shared dispatch QoS is downgraded to 0 + Msg = case SubQoS =:= ?QOS_0 of + true -> emqx_shared_sub:maybe_ack(Msg0); + false -> Msg0 + end, + run_dispatch_steps(Steps, Msg#message{qos = min(SubQoS, PubQoS)}, State); +run_dispatch_steps([{qos, SubQoS}|Steps], Msg = #message{qos = PubQoS}, State = #state{upgrade_qos = true}) -> + run_dispatch_steps(Steps, Msg#message{qos = max(SubQoS, PubQoS)}, State); +run_dispatch_steps([{rap, _Rap}|Steps], Msg = #message{flags = Flags, headers = #{retained := true}}, State = #state{}) -> + run_dispatch_steps(Steps, Msg#message{flags = maps:put(retain, true, Flags)}, State); +run_dispatch_steps([{rap, 0}|Steps], Msg = #message{flags = Flags}, State = #state{}) -> + run_dispatch_steps(Steps, Msg#message{flags = maps:put(retain, false, Flags)}, State); +run_dispatch_steps([{rap, _}|Steps], Msg, State) -> + run_dispatch_steps(Steps, Msg, State); +run_dispatch_steps([{subid, SubId}|Steps], Msg, State) -> + run_dispatch_steps(Steps, emqx_message:set_header('Subscription-Identifier', SubId, Msg), State). + +%% Enqueue message if the client has been disconnected +dispatch(Msg, State = #state{client_id = ClientId, conn_pid = undefined}) -> + case emqx_hooks:run('message.dropped', [#{client_id => ClientId}, Msg]) of + ok -> enqueue_msg(Msg, State); + stop -> State + end; + +%% Deliver qos0 message directly to client +dispatch(Msg = #message{qos = ?QOS_0} = Msg, State) -> + ok = deliver(undefined, Msg, State), + State; + +dispatch(Msg = #message{qos = QoS} = Msg, + State = #state{next_pkt_id = PacketId, inflight = Inflight}) + when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 -> + case emqx_inflight:is_full(Inflight) of + true -> + enqueue_msg(Msg, State); + false -> + ok = deliver(PacketId, Msg, State), + await(PacketId, Msg, next_pkt_id(State)) + end. + +enqueue_msg(Msg, State = #state{mqueue = Q}) -> + emqx_pd:update_counter(enqueue_stats, 1), + {Dropped, NewQ} = emqx_mqueue:in(Msg, Q), + Dropped =/= undefined andalso emqx_shared_sub:maybe_nack_dropped(Dropped), + State#state{mqueue = NewQ}. + +%%------------------------------------------------------------------------------ +%% Deliver +%%------------------------------------------------------------------------------ + +redeliver({PacketId, Msg = #message{qos = QoS}}, State) -> + deliver(PacketId, if QoS =:= ?QOS_2 -> Msg; + true -> emqx_message:set_flag(dup, Msg) + end, State); + +redeliver({pubrel, PacketId}, #state{conn_pid = ConnPid}) -> + ConnPid ! {deliver, {pubrel, PacketId}}. + +deliver(PacketId, Msg, State) -> + emqx_pd:update_counter(deliver_stats, 1), + %% Ack QoS1/QoS2 messages when message is delivered to connection. + %% NOTE: NOT to wait for PUBACK because: + %% The sender is monitoring this session process, + %% if the message is delivered to client but connection or session crashes, + %% sender will try to dispatch the message to the next shared subscriber. + %% This violates spec as QoS2 messages are not allowed to be sent to more + %% than one member in the group. + do_deliver(PacketId, emqx_shared_sub:maybe_ack(Msg), State). + +do_deliver(PacketId, Msg, #state{conn_pid = ConnPid, binding = local}) -> + ConnPid ! {deliver, {publish, PacketId, Msg}}, ok; +do_deliver(PacketId, Msg, #state{conn_pid = ConnPid, binding = remote}) -> + emqx_rpc:cast(node(ConnPid), erlang, send, [ConnPid, {deliver, {publish, PacketId, Msg}}]). + +%%------------------------------------------------------------------------------ +%% Awaiting ACK for QoS1/QoS2 Messages +%%------------------------------------------------------------------------------ + +await(PacketId, Msg, State = #state{inflight = Inflight}) -> + Inflight1 = emqx_inflight:insert( + PacketId, {publish, {PacketId, Msg}, os:timestamp()}, Inflight), + ensure_retry_timer(State#state{inflight = Inflight1}). + +acked(puback, PacketId, State = #state{client_id = ClientId, inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, {publish, {_, Msg}, _Ts}} -> + emqx_hooks:run('message.acked', [#{client_id => ClientId}], Msg), + State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}; + none -> + ?LOG(warning, "Duplicated PUBACK PacketId ~w", [PacketId], State), + State + end; + +acked(pubrec, PacketId, State = #state{client_id = ClientId, inflight = Inflight}) -> + case emqx_inflight:lookup(PacketId, Inflight) of + {value, {publish, {_, Msg}, _Ts}} -> + emqx_hooks:run('message.acked', [#{client_id => ClientId}], Msg), + State#state{inflight = emqx_inflight:update(PacketId, {pubrel, PacketId, os:timestamp()}, Inflight)}; + {value, {pubrel, PacketId, _Ts}} -> + ?LOG(warning, "Duplicated PUBREC PacketId ~w", [PacketId], State), + State; + none -> + ?LOG(warning, "Unexpected PUBREC PacketId ~w", [PacketId], State), + State + end; + +acked(pubcomp, PacketId, State = #state{inflight = Inflight}) -> + State#state{inflight = emqx_inflight:delete(PacketId, Inflight)}. + +%%------------------------------------------------------------------------------ +%% Dequeue +%%------------------------------------------------------------------------------ + +%% Do nothing if client is disconnected +dequeue(State = #state{conn_pid = undefined}) -> + State; + +dequeue(State = #state{inflight = Inflight}) -> + case emqx_inflight:is_full(Inflight) of + true -> State; + false -> dequeue2(State) + end. + +dequeue2(State = #state{mqueue = Q}) -> + case emqx_mqueue:out(Q) of + {empty, _Q} -> State; + {{value, Msg}, Q1} -> + %% Dequeue more + dequeue(dispatch(Msg, State#state{mqueue = Q1})) + end. + +%%------------------------------------------------------------------------------ +%% Ensure timers + +ensure_await_rel_timer(State = #state{await_rel_timer = undefined, await_rel_timeout = Timeout}) -> + ensure_await_rel_timer(Timeout, State); + +ensure_await_rel_timer(State) -> + State. + +ensure_await_rel_timer(Timeout, State = #state{await_rel_timer = undefined}) -> + State#state{await_rel_timer = emqx_misc:start_timer(Timeout, check_awaiting_rel)}; +ensure_await_rel_timer(_Timeout, State) -> + State. + +ensure_retry_timer(State = #state{retry_timer = undefined, retry_interval = Interval}) -> + ensure_retry_timer(Interval, State); +ensure_retry_timer(State) -> + State. + +ensure_retry_timer(Interval, State = #state{retry_timer = undefined}) -> + State#state{retry_timer = emqx_misc:start_timer(Interval, retry_delivery)}; +ensure_retry_timer(_Timeout, State) -> + State. + +ensure_expire_timer(State = #state{expiry_interval = Interval}) when Interval > 0 andalso Interval =/= 16#ffffffff -> + State#state{expiry_timer = emqx_misc:start_timer(Interval * 1000, expired)}; +ensure_expire_timer(State) -> + State. + +ensure_will_delay_timer(State = #state{will_msg = #message{headers = #{'Will-Delay-Interval' := WillDelayInterval}}}) -> + State#state{will_delay_timer = emqx_misc:start_timer(WillDelayInterval * 1000, will_delay)}; +ensure_will_delay_timer(State = #state{will_msg = WillMsg}) -> + send_willmsg(WillMsg), + State#state{will_msg = undefined}. + +ensure_stats_timer(State = #state{enable_stats = true, + stats_timer = undefined, + idle_timeout = IdleTimeout}) -> + State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; +ensure_stats_timer(State) -> + State. + +%%------------------------------------------------------------------------------ +%% Next Packet Id + +next_pkt_id(State = #state{next_pkt_id = 16#FFFF}) -> + State#state{next_pkt_id = 1}; + +next_pkt_id(State = #state{next_pkt_id = Id}) -> + State#state{next_pkt_id = Id + 1}. + +%% Take only the payload size into account, add other fields if necessary +msg_size(#message{payload = Payload}) -> payload_size(Payload). + +%% Payload should be binary(), but not 100% sure. Need dialyzer! +payload_size(Payload) -> erlang:iolist_size(Payload). + +%%------------------------------------------------------------------------------ +%% Maybe GC + +maybe_gc(_, State = #state{gc_state = undefined}) -> + State; +maybe_gc({Cnt, Oct}, State = #state{gc_state = GCSt}) -> + {_, GCSt1} = emqx_gc:run(Cnt, Oct, GCSt), + State#state{gc_state = GCSt1}. + +%%------------------------------------------------------------------------------ +%% Helper functions + +reply({Reply, State}) -> + reply(Reply, State). + +reply(Reply, State) -> + {reply, Reply, State}. + +noreply(State) -> + {noreply, State}. + +shutdown(Reason, State) -> + {stop, {shutdown, Reason}, State}. + diff --git a/src/emqx_session_sup.erl b/src/emqx_session_sup.erl new file mode 100644 index 000000000..efa64815b --- /dev/null +++ b/src/emqx_session_sup.erl @@ -0,0 +1,256 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_session_sup). + +-behaviour(gen_server). + +-export([start_link/1]). +-export([start_session/1, count_sessions/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-type(shutdown() :: brutal_kill | infinity | pos_integer()). + +-record(state, { + sessions :: #{pid() => emqx_types:client_id()}, + mfargs :: mfa(), + shutdown :: shutdown(), + clean_down :: fun() + }). + +-define(SUP, ?MODULE). +-define(BATCH_EXIT, 100000). +-define(ERROR_MSG(Format, Args), + error_logger:error_msg("[~s] " ++ Format, [?MODULE | Args])). + +%% @doc Start session supervisor. +-spec(start_link(map()) -> emqx_types:startlink_ret()). +start_link(SessSpec) when is_map(SessSpec) -> + gen_server:start_link({local, ?SUP}, ?MODULE, [SessSpec], []). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% @doc Start a session. +-spec(start_session(map()) -> emqx_types:startlink_ret()). +start_session(SessAttrs) -> + gen_server:call(?SUP, {start_session, SessAttrs}, infinity). + +%% @doc Count sessions. +-spec(count_sessions() -> non_neg_integer()). +count_sessions() -> + gen_server:call(?SUP, count_sessions, infinity). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([Spec]) -> + process_flag(trap_exit, true), + MFA = maps:get(start, Spec), + Shutdown = maps:get(shutdown, Spec, brutal_kill), + CleanDown = maps:get(clean_down, Spec, undefined), + State = #state{sessions = #{}, + mfargs = MFA, + shutdown = Shutdown, + clean_down = CleanDown + }, + {ok, State}. + +handle_call({start_session, SessAttrs = #{client_id := ClientId}}, _From, + State = #state{sessions = SessMap, mfargs = {M, F, Args}}) -> + try erlang:apply(M, F, [SessAttrs | Args]) of + {ok, Pid} -> + reply({ok, Pid}, State#state{sessions = maps:put(Pid, ClientId, SessMap)}); + ignore -> + reply(ignore, State); + {error, Reason} -> + reply({error, Reason}, State) + catch + _:Error:Stk -> + ?ERROR_MSG("Failed to start session ~p: ~p, stacktrace:~n~p", + [ClientId, Error, Stk]), + reply({error, Error}, State) + end; + +handle_call(count_sessions, _From, State = #state{sessions = SessMap}) -> + {reply, maps:size(SessMap), State}; + +handle_call(Req, _From, State) -> + ?ERROR_MSG("unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + ?ERROR_MSG("unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'EXIT', Pid, _Reason}, State = #state{sessions = SessMap, clean_down = CleanDown}) -> + SessPids = [Pid | drain_exit(?BATCH_EXIT, [])], + {SessItems, SessMap1} = erase_all(SessPids, SessMap), + (CleanDown =:= undefined) + orelse emqx_pool:async_submit( + fun lists:foreach/2, [CleanDown, SessItems]), + {noreply, State#state{sessions = SessMap1}}; + +handle_info(Info, State) -> + ?ERROR_MSG("unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, State) -> + terminate_children(State). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +drain_exit(0, Acc) -> + lists:reverse(Acc); +drain_exit(Cnt, Acc) -> + receive + {'EXIT', Pid, _Reason} -> + drain_exit(Cnt - 1, [Pid|Acc]) + after 0 -> + lists:reverse(Acc) + end. + +erase_all(Pids, Map) -> + lists:foldl( + fun(Pid, {Acc, M}) -> + case maps:take(Pid, M) of + {Val, M1} -> + {[{Val, Pid}|Acc], M1}; + error -> + {Acc, M} + end + end, {[], Map}, Pids). + +terminate_children(State = #state{sessions = SessMap, shutdown = Shutdown}) -> + {Pids, EStack0} = monitor_children(SessMap), + Sz = sets:size(Pids), + EStack = + case Shutdown of + brutal_kill -> + sets:fold(fun(P, _) -> exit(P, kill) end, ok, Pids), + wait_children(Shutdown, Pids, Sz, undefined, EStack0); + infinity -> + sets:fold(fun(P, _) -> exit(P, shutdown) end, ok, Pids), + wait_children(Shutdown, Pids, Sz, undefined, EStack0); + Time when is_integer(Time) -> + sets:fold(fun(P, _) -> exit(P, shutdown) end, ok, Pids), + TRef = erlang:start_timer(Time, self(), kill), + wait_children(Shutdown, Pids, Sz, TRef, EStack0) + end, + %% Unroll stacked errors and report them + dict:fold(fun(Reason, Pid, _) -> + report_error(connection_shutdown_error, Reason, Pid, State) + end, ok, EStack). + +monitor_children(SessMap) -> + lists:foldl( + fun(Pid, {Pids, EStack}) -> + case monitor_child(Pid) of + ok -> + {sets:add_element(Pid, Pids), EStack}; + {error, normal} -> + {Pids, EStack}; + {error, Reason} -> + {Pids, dict:append(Reason, Pid, EStack)} + end + end, {sets:new(), dict:new()}, maps:keys(SessMap)). + +%% Help function to shutdown/2 switches from link to monitor approach +monitor_child(Pid) -> + %% Do the monitor operation first so that if the child dies + %% before the monitoring is done causing a 'DOWN'-message with + %% reason noproc, we will get the real reason in the 'EXIT'-message + %% unless a naughty child has already done unlink... + erlang:monitor(process, Pid), + unlink(Pid), + + receive + %% If the child dies before the unlik we must empty + %% the mail-box of the 'EXIT'-message and the 'DOWN'-message. + {'EXIT', Pid, Reason} -> + receive + {'DOWN', _, process, Pid, _} -> + {error, Reason} + end + after 0 -> + %% If a naughty child did unlink and the child dies before + %% monitor the result will be that shutdown/2 receives a + %% 'DOWN'-message with reason noproc. + %% If the child should die after the unlink there + %% will be a 'DOWN'-message with a correct reason + %% that will be handled in shutdown/2. + ok + end. + +wait_children(_Shutdown, _Pids, 0, undefined, EStack) -> + EStack; +wait_children(_Shutdown, _Pids, 0, TRef, EStack) -> + %% If the timer has expired before its cancellation, we must empty the + %% mail-box of the 'timeout'-message. + erlang:cancel_timer(TRef), + receive + {timeout, TRef, kill} -> + EStack + after 0 -> + EStack + end; + +%%TODO: Copied from supervisor.erl, rewrite it later. +wait_children(brutal_kill, Pids, Sz, TRef, EStack) -> + receive + {'DOWN', _MRef, process, Pid, killed} -> + wait_children(brutal_kill, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + + {'DOWN', _MRef, process, Pid, Reason} -> + wait_children(brutal_kill, sets:del_element(Pid, Pids), + Sz-1, TRef, dict:append(Reason, Pid, EStack)) + end; + +wait_children(Shutdown, Pids, Sz, TRef, EStack) -> + receive + {'DOWN', _MRef, process, Pid, shutdown} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + {'DOWN', _MRef, process, Pid, normal} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, TRef, EStack); + {'DOWN', _MRef, process, Pid, Reason} -> + wait_children(Shutdown, sets:del_element(Pid, Pids), Sz-1, + TRef, dict:append(Reason, Pid, EStack)); + {timeout, TRef, kill} -> + sets:fold(fun(P, _) -> exit(P, kill) end, ok, Pids), + wait_children(Shutdown, Pids, Sz-1, undefined, EStack) + end. + +report_error(Error, Reason, Pid, #state{mfargs = MFA}) -> + SupName = list_to_atom("esockd_connection_sup - " ++ pid_to_list(self())), + ErrorMsg = [{supervisor, SupName}, + {errorContext, Error}, + {reason, Reason}, + {offender, [{pid, Pid}, + {name, connection}, + {mfargs, MFA}]}], + error_logger:error_report(supervisor_report, ErrorMsg). + +reply(Repy, State) -> + {reply, Repy, State}. + diff --git a/src/emqx_shared_sub.erl b/src/emqx_shared_sub.erl new file mode 100644 index 000000000..b7e41213b --- /dev/null +++ b/src/emqx_shared_sub.erl @@ -0,0 +1,353 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_shared_sub). + +-behaviour(gen_server). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +%% Mnesia bootstrap +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + +-export([start_link/0]). + +-export([subscribe/3, unsubscribe/3]). +-export([dispatch/3]). +-export([maybe_ack/1, maybe_nack_dropped/1, nack_no_connection/1, is_ack_required/1]). + +%% for testing +-export([subscribers/2]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(SERVER, ?MODULE). +-define(TAB, emqx_shared_subscription). +-define(SHARED_SUBS, emqx_shared_subscriber). +-define(ALIVE_SUBS, emqx_alive_shared_subscribers). +-define(SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS, 5). +-define(ack, shared_sub_ack). +-define(nack(Reason), {shared_sub_nack, Reason}). +-define(IS_LOCAL_PID(Pid), (is_pid(Pid) andalso node(Pid) =:= node())). +-define(no_ack, no_ack). + +-record(state, {pmon}). +-record(emqx_shared_subscription, {group, topic, subpid}). + +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +mnesia(boot) -> + ok = ekka_mnesia:create_table(?TAB, [ + {type, bag}, + {ram_copies, [node()]}, + {record_name, emqx_shared_subscription}, + {attributes, record_info(fields, emqx_shared_subscription)}]); + +mnesia(copy) -> + ok = ekka_mnesia:copy_table(?TAB). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +subscribe(Group, Topic, SubPid) when is_pid(SubPid) -> + gen_server:call(?SERVER, {subscribe, Group, Topic, SubPid}). + +unsubscribe(Group, Topic, SubPid) when is_pid(SubPid) -> + gen_server:call(?SERVER, {unsubscribe, Group, Topic, SubPid}). + +record(Group, Topic, SubPid) -> + #emqx_shared_subscription{group = Group, topic = Topic, subpid = SubPid}. + +dispatch(Group, Topic, Delivery) -> + dispatch(Group, Topic, Delivery, _FailedSubs = []). + +dispatch(Group, Topic, Delivery = #delivery{message = Msg, results = Results}, FailedSubs) -> + #message{from = ClientId} = Msg, + case pick(strategy(), ClientId, Group, Topic, FailedSubs) of + false -> + Delivery; + SubPid -> + case do_dispatch(SubPid, Topic, Msg) of + ok -> + Delivery#delivery{results = [{dispatch, {Group, Topic}, 1} | Results]}; + {error, _Reason} -> + %% failed to dispatch to this sub, try next + %% 'Reason' is discarded so far, meaning for QoS1/2 messages + %% if all subscribers are off line, the dispatch would faile + %% even if there are sessions not expired yet. + %% If required, we can make use of the 'no_connection' reason to perform + %% retry without requiring acks, so the messages can be delivered + %% to sessions of offline clients + dispatch(Group, Topic, Delivery, [SubPid | FailedSubs]) + end + end. + +-spec(strategy() -> random | round_robin | sticky | hash). +strategy() -> + emqx_config:get_env(shared_subscription_strategy, round_robin). + +-spec(ack_enabled() -> boolean()). +ack_enabled() -> + emqx_config:get_env(shared_dispatch_ack_enabled, false). + +do_dispatch(SubPid, Topic, Msg) when SubPid =:= self() -> + %% Deadlock otherwise + _ = erlang:send(SubPid, {dispatch, Topic, Msg}), + ok; +do_dispatch(SubPid, Topic, Msg) -> + dispatch_per_qos(SubPid, Topic, Msg). + +%% return either 'ok' (when everything is fine) or 'error' +dispatch_per_qos(SubPid, Topic, #message{qos = ?QOS_0} = Msg) -> + %% For QoS 0 message, send it as regular dispatch + _ = erlang:send(SubPid, {dispatch, Topic, Msg}), + ok; +dispatch_per_qos(SubPid, Topic, Msg) -> + case ack_enabled() of + true -> + dispatch_with_ack(SubPid, Topic, Msg); + false -> + _ = erlang:send(SubPid, {dispatch, Topic, Msg}), + ok + end. + +dispatch_with_ack(SubPid, Topic, Msg) -> + %% For QoS 1/2 message, expect an ack + Ref = erlang:monitor(process, SubPid), + Sender = self(), + _ = erlang:send(SubPid, {dispatch, Topic, with_ack_ref(Msg, {Sender, Ref})}), + Timeout = case Msg#message.qos of + ?QOS_1 -> timer:seconds(?SHARED_SUB_QOS1_DISPATCH_TIMEOUT_SECONDS); + ?QOS_2 -> infinity + end, + try + receive + {Ref, ?ack} -> + ok; + {Ref, ?nack(Reason)} -> + %% the receive session may nack this message when its queue is full + {error, Reason}; + {'DOWN', Ref, process, SubPid, Reason} -> + {error, Reason} + after + Timeout -> + {error, timeout} + end + after + _ = erlang:demonitor(Ref, [flush]) + end. + +with_ack_ref(Msg, SenderRef) -> + emqx_message:set_headers(#{shared_dispatch_ack => SenderRef}, Msg). + +without_ack_ref(Msg) -> + emqx_message:set_headers(#{shared_dispatch_ack => ?no_ack}, Msg). + +get_ack_ref(Msg) -> + emqx_message:get_header(shared_dispatch_ack, Msg, ?no_ack). + +-spec(is_ack_required(emqx_types:message()) -> boolean()). +is_ack_required(Msg) -> ?no_ack =/= get_ack_ref(Msg). + +%% @doc Negative ack dropped message due to message queue being full. +-spec(maybe_nack_dropped(emqx_types:message()) -> ok). +maybe_nack_dropped(Msg) -> + case get_ack_ref(Msg) of + ?no_ack -> ok; + {Sender, Ref} -> nack(Sender, Ref, drpped) + end. + +%% @doc Negative ack message due to connection down. +%% Assuming this function is always called when ack is required +%% i.e is_ack_required returned true. +-spec(nack_no_connection(emqx_types:message()) -> ok). +nack_no_connection(Msg) -> + {Sender, Ref} = get_ack_ref(Msg), + nack(Sender, Ref, no_connection). + +-spec(nack(pid(), reference(), dropped | no_connection) -> ok). +nack(Sender, Ref, Reason) -> + erlang:send(Sender, {Ref, ?nack(Reason)}), + ok. + +-spec(maybe_ack(emqx_types:message()) -> emqx_types:message()). +maybe_ack(Msg) -> + case get_ack_ref(Msg) of + ?no_ack -> + Msg; + {Sender, Ref} -> + erlang:send(Sender, {Ref, ?ack}), + without_ack_ref(Msg) + end. + +pick(sticky, ClientId, Group, Topic, FailedSubs) -> + Sub0 = erlang:get({shared_sub_sticky, Group, Topic}), + case is_active_sub(Sub0, FailedSubs) of + true -> + %% the old subscriber is still alive + %% keep using it for sticky strategy + Sub0; + false -> + %% randomly pick one for the first message + Sub = do_pick(random, ClientId, Group, Topic, FailedSubs), + %% stick to whatever pick result + erlang:put({shared_sub_sticky, Group, Topic}, Sub), + Sub + end; +pick(Strategy, ClientId, Group, Topic, FailedSubs) -> + do_pick(Strategy, ClientId, Group, Topic, FailedSubs). + +do_pick(Strategy, ClientId, Group, Topic, FailedSubs) -> + case subscribers(Group, Topic) -- FailedSubs of + [] -> false; + [Sub] -> Sub; + All -> pick_subscriber(Group, Topic, Strategy, ClientId, All) + end. + +pick_subscriber(Group, Topic, Strategy, ClientId, Subs) -> + Nth = do_pick_subscriber(Group, Topic, Strategy, ClientId, length(Subs)), + lists:nth(Nth, Subs). + +do_pick_subscriber(_Group, _Topic, random, _ClientId, Count) -> + rand:uniform(Count); +do_pick_subscriber(_Group, _Topic, hash, ClientId, Count) -> + 1 + erlang:phash2(ClientId) rem Count; +do_pick_subscriber(Group, Topic, round_robin, _ClientId, Count) -> + Rem = case erlang:get({shared_sub_round_robin, Group, Topic}) of + undefined -> 0; + N -> (N + 1) rem Count + end, + _ = erlang:put({shared_sub_round_robin, Group, Topic}, Rem), + Rem + 1. + +subscribers(Group, Topic) -> + ets:select(?TAB, [{{emqx_shared_subscription, Group, Topic, '$1'}, [], ['$1']}]). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + mnesia:subscribe({table, ?TAB, simple}), + {atomic, PMon} = mnesia:transaction(fun init_monitors/0), + ok = emqx_tables:new(?SHARED_SUBS, [protected, bag]), + ok = emqx_tables:new(?ALIVE_SUBS, [protected, set, {read_concurrency, true}]), + {ok, update_stats(#state{pmon = PMon})}. + +init_monitors() -> + mnesia:foldl( + fun(#emqx_shared_subscription{subpid = SubPid}, Mon) -> + emqx_pmon:monitor(SubPid, Mon) + end, emqx_pmon:new(), ?TAB). + +handle_call({subscribe, Group, Topic, SubPid}, _From, State = #state{pmon = PMon}) -> + mnesia:dirty_write(?TAB, record(Group, Topic, SubPid)), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:do_add_route(Topic, {Group, node()}) + end, + ok = maybe_insert_alive_tab(SubPid), + true = ets:insert(?SHARED_SUBS, {{Group, Topic}, SubPid}), + {reply, ok, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; + +handle_call({unsubscribe, Group, Topic, SubPid}, _From, State) -> + mnesia:dirty_delete_object(?TAB, record(Group, Topic, SubPid)), + true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) + end, + {reply, ok, State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[SharedSub] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[SharedSub] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({mnesia_table_event, {write, NewRecord, _}}, State = #state{pmon = PMon}) -> + #emqx_shared_subscription{subpid = SubPid} = NewRecord, + {noreply, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; + +handle_info({mnesia_table_event, {delete_object, OldRecord, _}}, State = #state{pmon = PMon}) -> + #emqx_shared_subscription{subpid = SubPid} = OldRecord, + {noreply, update_stats(State#state{pmon = emqx_pmon:demonitor(SubPid, PMon)})}; + +handle_info({mnesia_table_event, _Event}, State) -> + {noreply, State}; + +handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{pmon = PMon}) -> + emqx_logger:info("[SharedSub] shared subscriber down: ~p", [SubPid]), + cleanup_down(SubPid), + {noreply, update_stats(State#state{pmon = emqx_pmon:erase(SubPid, PMon)})}; + +handle_info(Info, State) -> + emqx_logger:error("[SharedSub] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + mnesia:unsubscribe({table, ?TAB, simple}). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +%% keep track of alive remote pids +maybe_insert_alive_tab(Pid) when ?IS_LOCAL_PID(Pid) -> ok; +maybe_insert_alive_tab(Pid) when is_pid(Pid) -> ets:insert(?ALIVE_SUBS, {Pid}), ok. + +cleanup_down(SubPid) -> + ?IS_LOCAL_PID(SubPid) orelse ets:delete(?ALIVE_SUBS, SubPid), + lists:foreach( + fun(Record = #emqx_shared_subscription{topic = Topic, group = Group}) -> + ok = mnesia:dirty_delete_object(?TAB, Record), + true = ets:delete_object(?SHARED_SUBS, {{Group, Topic}, SubPid}), + case ets:member(?SHARED_SUBS, {Group, Topic}) of + true -> ok; + false -> ok = emqx_router:do_delete_route(Topic, {Group, node()}) + end + end, mnesia:dirty_match_object(#emqx_shared_subscription{_ = '_', subpid = SubPid})). + +update_stats(State) -> + emqx_stats:setstat('subscriptions/shared/count', 'subscriptions/shared/max', ets:info(?TAB, size)), + State. + +%% Return 'true' if the subscriber process is alive AND not in the failed list +is_active_sub(Pid, FailedSubs) -> + is_alive_sub(Pid) andalso not lists:member(Pid, FailedSubs). + +%% erlang:is_process_alive/1 does not work with remote pid. +is_alive_sub(Pid) when ?IS_LOCAL_PID(Pid) -> + erlang:is_process_alive(Pid); +is_alive_sub(Pid) -> + [] =/= ets:lookup(?ALIVE_SUBS, Pid). + diff --git a/src/emqx_sm.erl b/src/emqx_sm.erl new file mode 100644 index 000000000..ab270f1af --- /dev/null +++ b/src/emqx_sm.erl @@ -0,0 +1,267 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sm). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/0]). + +-export([open_session/1, close_session/1]). +-export([resume_session/2]). +-export([discard_session/1, discard_session/2]). +-export([register_session/1, register_session/2]). +-export([unregister_session/1, unregister_session/2]). +-export([get_session_attrs/1, get_session_attrs/2, + set_session_attrs/2, set_session_attrs/3]). +-export([get_session_stats/1, get_session_stats/2, + set_session_stats/2, set_session_stats/3]). +-export([lookup_session_pids/1]). + +%% Internal functions for rpc +-export([dispatch/3]). + +%% Internal function for stats +-export([stats_fun/0]). + +%% Internal function for emqx_session_sup +-export([clean_down/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(SM, ?MODULE). + +%% ETS Tables for session management. +-define(SESSION_TAB, emqx_session). +-define(SESSION_P_TAB, emqx_session_p). +-define(SESSION_ATTRS_TAB, emqx_session_attrs). +-define(SESSION_STATS_TAB, emqx_session_stats). + +-define(BATCH_SIZE, 100000). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?SM}, ?MODULE, [], []). + +%% @doc Open a session. +-spec(open_session(map()) -> {ok, pid()} | {ok, pid(), boolean()} | {error, term()}). +open_session(SessAttrs = #{clean_start := true, client_id := ClientId, conn_pid := ConnPid}) -> + CleanStart = fun(_) -> + ok = discard_session(ClientId, ConnPid), + emqx_session_sup:start_session(SessAttrs) + end, + emqx_sm_locker:trans(ClientId, CleanStart); + +open_session(SessAttrs = #{clean_start := false, client_id := ClientId}) -> + ResumeStart = fun(_) -> + case resume_session(ClientId, SessAttrs) of + {ok, SessPid} -> + {ok, SessPid, true}; + {error, not_found} -> + emqx_session_sup:start_session(SessAttrs) + end + end, + emqx_sm_locker:trans(ClientId, ResumeStart). + +%% @doc Discard all the sessions identified by the ClientId. +-spec(discard_session(emqx_types:client_id()) -> ok). +discard_session(ClientId) when is_binary(ClientId) -> + discard_session(ClientId, self()). + +-spec(discard_session(emqx_types:client_id(), pid()) -> ok). +discard_session(ClientId, ConnPid) when is_binary(ClientId) -> + lists:foreach( + fun(SessPid) -> + try emqx_session:discard(SessPid, ConnPid) + catch + _:Error:_Stk -> + emqx_logger:error("[SM] Failed to discard ~p: ~p", [SessPid, Error]) + end + end, lookup_session_pids(ClientId)). + +%% @doc Try to resume a session. +-spec(resume_session(emqx_types:client_id(), map()) -> {ok, pid()} | {error, term()}). +resume_session(ClientId, SessAttrs = #{conn_pid := ConnPid}) -> + case lookup_session_pids(ClientId) of + [] -> {error, not_found}; + [SessPid] -> + ok = emqx_session:resume(SessPid, SessAttrs), + {ok, SessPid}; + SessPids -> + [SessPid|StalePids] = lists:reverse(SessPids), + emqx_logger:error("[SM] More than one session found: ~p", [SessPids]), + lists:foreach(fun(StalePid) -> + catch emqx_session:discard(StalePid, ConnPid) + end, StalePids), + ok = emqx_session:resume(SessPid, SessAttrs), + {ok, SessPid} + end. + +%% @doc Close a session. +-spec(close_session(emqx_types:client_id() | pid()) -> ok). +close_session(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> ok; + [SessPid] -> close_session(SessPid); + SessPids -> lists:foreach(fun close_session/1, SessPids) + end; + +close_session(SessPid) when is_pid(SessPid) -> + emqx_session:close(SessPid). + +%% @doc Register a session. +-spec(register_session(emqx_types:client_id()) -> ok). +register_session(ClientId) when is_binary(ClientId) -> + register_session(ClientId, self()). + +-spec(register_session(emqx_types:client_id(), pid()) -> ok). +register_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + Session = {ClientId, SessPid}, + true = ets:insert(?SESSION_TAB, Session), + emqx_sm_registry:register_session(Session). + +%% @doc Unregister a session +-spec(unregister_session(emqx_types:client_id()) -> ok). +unregister_session(ClientId) when is_binary(ClientId) -> + unregister_session(ClientId, self()). + +-spec(unregister_session(emqx_types:client_id(), pid()) -> ok). +unregister_session(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + Session = {ClientId, SessPid}, + true = ets:delete(?SESSION_STATS_TAB, Session), + true = ets:delete(?SESSION_ATTRS_TAB, Session), + true = ets:delete_object(?SESSION_P_TAB, Session), + true = ets:delete_object(?SESSION_TAB, Session), + emqx_sm_registry:unregister_session(Session). + +%% @doc Get session attrs +-spec(get_session_attrs(emqx_types:client_id()) -> list(emqx_session:attr())). +get_session_attrs(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> []; + [SessPid|_] -> get_session_attrs(ClientId, SessPid) + end. + +-spec(get_session_attrs(emqx_types:client_id(), pid()) -> list(emqx_session:attr())). +get_session_attrs(ClientId, SessPid) when is_binary(ClientId), is_pid(SessPid) -> + emqx_tables:lookup_value(?SESSION_ATTRS_TAB, {ClientId, SessPid}, []). + +%% @doc Set session attrs +-spec(set_session_attrs(emqx_types:client_id(), list(emqx_session:attr())) -> true). +set_session_attrs(ClientId, SessAttrs) when is_binary(ClientId) -> + set_session_attrs(ClientId, self(), SessAttrs). + +-spec(set_session_attrs(emqx_types:client_id(), pid(), list(emqx_session:attr())) -> true). +set_session_attrs(ClientId, SessPid, SessAttrs) when is_binary(ClientId), is_pid(SessPid) -> + Session = {ClientId, SessPid}, + true = ets:insert(?SESSION_ATTRS_TAB, {Session, SessAttrs}), + proplists:get_value(clean_start, SessAttrs, true) orelse ets:insert(?SESSION_P_TAB, Session). + +%% @doc Get session stats +-spec(get_session_stats(emqx_types:client_id()) -> list(emqx_stats:stats())). +get_session_stats(ClientId) when is_binary(ClientId) -> + case lookup_session_pids(ClientId) of + [] -> []; + [SessPid|_] -> + get_session_stats(ClientId, SessPid) + end. + +-spec(get_session_stats(emqx_types:client_id(), pid()) -> list(emqx_stats:stats())). +get_session_stats(ClientId, SessPid) when is_binary(ClientId) -> + emqx_tables:lookup_value(?SESSION_STATS_TAB, {ClientId, SessPid}, []). + +%% @doc Set session stats +-spec(set_session_stats(emqx_types:client_id(), emqx_stats:stats()) -> true). +set_session_stats(ClientId, Stats) when is_binary(ClientId) -> + set_session_stats(ClientId, self(), Stats). + +-spec(set_session_stats(emqx_types:client_id(), pid(), emqx_stats:stats()) -> true). +set_session_stats(ClientId, SessPid, Stats) when is_binary(ClientId), is_pid(SessPid) -> + ets:insert(?SESSION_STATS_TAB, {{ClientId, SessPid}, Stats}). + +%% @doc Lookup session pid. +-spec(lookup_session_pids(emqx_types:client_id()) -> list(pid())). +lookup_session_pids(ClientId) -> + case emqx_sm_registry:is_enabled() of + true -> emqx_sm_registry:lookup_session(ClientId); + false -> emqx_tables:lookup_value(?SESSION_TAB, ClientId, []) + end. + +%% @doc Dispatch a message to the session. +-spec(dispatch(emqx_types:client_id(), emqx_topic:topic(), emqx_types:message()) -> any()). +dispatch(ClientId, Topic, Msg) -> + case lookup_session_pids(ClientId) of + [SessPid|_] when is_pid(SessPid) -> + SessPid ! {dispatch, Topic, Msg}; + [] -> + emqx_hooks:run('message.dropped', [#{client_id => ClientId}, Msg]) + end. + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + TabOpts = [public, set, {write_concurrency, true}], + ok = emqx_tables:new(?SESSION_TAB, [{read_concurrency, true} | TabOpts]), + ok = emqx_tables:new(?SESSION_P_TAB, TabOpts), + ok = emqx_tables:new(?SESSION_ATTRS_TAB, TabOpts), + ok = emqx_tables:new(?SESSION_STATS_TAB, TabOpts), + ok = emqx_stats:update_interval(sess_stats, fun ?MODULE:stats_fun/0), + {ok, #{}}. + +handle_call(Req, _From, State) -> + emqx_logger:error("[SM] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[SM] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + emqx_logger:error("[SM] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + emqx_stats:cancel_update(sess_stats). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +clean_down(Session = {ClientId, SessPid}) -> + case ets:member(?SESSION_TAB, ClientId) + orelse ets:member(?SESSION_ATTRS_TAB, Session) of + true -> + unregister_session(ClientId, SessPid); + false -> ok + end. + +stats_fun() -> + safe_update_stats(?SESSION_TAB, 'sessions/count', 'sessions/max'), + safe_update_stats(?SESSION_P_TAB, 'sessions/persistent/count', 'sessions/persistent/max'). + +safe_update_stats(Tab, Stat, MaxStat) -> + case ets:info(Tab, size) of + undefined -> ok; + Size -> emqx_stats:setstat(Stat, MaxStat, Size) + end. + diff --git a/src/emqx_sm_locker.erl b/src/emqx_sm_locker.erl new file mode 100644 index 000000000..409331b88 --- /dev/null +++ b/src/emqx_sm_locker.erl @@ -0,0 +1,59 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sm_locker). + +-include("emqx.hrl"). + +-export([start_link/0]). + +-export([trans/2, trans/3]). +-export([lock/1, lock/2, unlock/1]). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + ekka_locker:start_link(?MODULE). + +-spec(trans(emqx_types:client_id(), fun(([node()]) -> any())) -> any()). +trans(ClientId, Fun) -> + trans(ClientId, Fun, undefined). + +-spec(trans(emqx_types:client_id() | undefined, + fun(([node()])-> any()), ekka_locker:piggyback()) -> any()). +trans(undefined, Fun, _Piggyback) -> + Fun([]); +trans(ClientId, Fun, Piggyback) -> + case lock(ClientId, Piggyback) of + {true, Nodes} -> + try Fun(Nodes) after unlock(ClientId) end; + {false, _Nodes} -> + {error, client_id_unavailable} + end. + +-spec(lock(emqx_types:client_id()) -> ekka_locker:lock_result()). +lock(ClientId) -> + ekka_locker:acquire(?MODULE, ClientId, strategy()). + +-spec(lock(emqx_types:client_id(), ekka_locker:piggyback()) -> ekka_locker:lock_result()). +lock(ClientId, Piggyback) -> + ekka_locker:acquire(?MODULE, ClientId, strategy(), Piggyback). + +-spec(unlock(emqx_types:client_id()) -> {boolean(), [node()]}). +unlock(ClientId) -> + ekka_locker:release(?MODULE, ClientId, strategy()). + +-spec(strategy() -> local | one | quorum | all). +strategy() -> + emqx_config:get_env(session_locking_strategy, quorum). + diff --git a/src/emqx_sm_registry.erl b/src/emqx_sm_registry.erl new file mode 100644 index 000000000..a7f44d771 --- /dev/null +++ b/src/emqx_sm_registry.erl @@ -0,0 +1,121 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sm_registry). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/0]). +-export([is_enabled/0]). +-export([register_session/1, lookup_session/1, unregister_session/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(REGISTRY, ?MODULE). +-define(TAB, emqx_session_registry). +-define(LOCK, {?MODULE, cleanup_sessions}). + +-record(global_session, {sid, pid}). + +-type(session_pid() :: pid()). + +%% @doc Start the global session manager. +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?REGISTRY}, ?MODULE, [], []). + +-spec(is_enabled() -> boolean()). +is_enabled() -> + emqx_config:get_env(enable_session_registry, true). + +-spec(lookup_session(emqx_types:client_id()) -> list(session_pid())). +lookup_session(ClientId) -> + [SessPid || #global_session{pid = SessPid} <- mnesia:dirty_read(?TAB, ClientId)]. + +-spec(register_session({emqx_types:client_id(), session_pid()}) -> ok). +register_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) -> + case is_enabled() of + true -> mnesia:dirty_write(?TAB, record(ClientId, SessPid)); + false -> ok + end. + +-spec(unregister_session({emqx_types:client_id(), session_pid()}) -> ok). +unregister_session({ClientId, SessPid}) when is_binary(ClientId), is_pid(SessPid) -> + case is_enabled() of + true -> mnesia:dirty_delete_object(?TAB, record(ClientId, SessPid)); + false -> ok + end. + +record(ClientId, SessPid) -> + #global_session{sid = ClientId, pid = SessPid}. + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + ok = ekka_mnesia:create_table(?TAB, [ + {type, bag}, + {ram_copies, [node()]}, + {record_name, global_session}, + {attributes, record_info(fields, global_session)}, + {storage_properties, [{ets, [{read_concurrency, true}, + {write_concurrency, true}]}]}]), + ok = ekka_mnesia:copy_table(?TAB), + ok = ekka:monitor(membership), + {ok, #{}}. + +handle_call(Req, _From, State) -> + emqx_logger:error("[Registry] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[Registry] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({membership, {mnesia, down, Node}}, State) -> + global:trans({?LOCK, self()}, + fun() -> + mnesia:transaction(fun cleanup_sessions/1, [Node]) + end), + {noreply, State}; + +handle_info({membership, _Event}, State) -> + {noreply, State}; + +handle_info(Info, State) -> + emqx_logger:error("[Registry] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +cleanup_sessions(Node) -> + Pat = [{#global_session{pid = '$1', _ = '_'}, [{'==', {node, '$1'}, Node}], ['$_']}], + lists:foreach(fun delete_session/1, mnesia:select(?TAB, Pat, write)). + +delete_session(Session) -> + mnesia:delete_object(?TAB, Session, write). + diff --git a/src/emqx_sm_sup.erl b/src/emqx_sm_sup.erl new file mode 100644 index 000000000..ed491c67f --- /dev/null +++ b/src/emqx_sm_sup.erl @@ -0,0 +1,64 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sm_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + %% Session locker + Locker = #{id => locker, + start => {emqx_sm_locker, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_sm_locker] + }, + %% Session registry + Registry = #{id => registry, + start => {emqx_sm_registry, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_sm_registry] + }, + %% Session Manager + Manager = #{id => manager, + start => {emqx_sm, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_sm] + }, + %% Session Sup + SessSpec = #{start => {emqx_session, start_link, []}, + shutdown => brutal_kill, + clean_down => fun emqx_sm:clean_down/1 + }, + SessionSup = #{id => session_sup, + start => {emqx_session_sup, start_link, [SessSpec ]}, + restart => transient, + shutdown => infinity, + type => supervisor, + modules => [emqx_session_sup] + }, + {ok, {{rest_for_one, 10, 3600}, [Locker, Registry, Manager, SessionSup]}}. + diff --git a/src/emqx_stats.erl b/src/emqx_stats.erl new file mode 100644 index 000000000..790c397b9 --- /dev/null +++ b/src/emqx_stats.erl @@ -0,0 +1,232 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_stats). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/0, start_link/1, stop/0]). + +%% Stats API. +-export([getstats/0, getstat/1]). +-export([setstat/2, setstat/3]). +-export([statsfun/1, statsfun/2]). +-export([update_interval/2, update_interval/3, cancel_update/1]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-record(update, {name, countdown, interval, func}). +-record(state, {timer, updates :: [#update{}], + tick_ms :: timeout()}). + +-type(stats() :: list({atom(), non_neg_integer()})). + +-export_type([stats/0]). + +%% Connection stats +-define(CONNECTION_STATS, [ + 'connections/count', % current connections + 'connections/max' % maximum connections connected +]). + +%% Session stats +-define(SESSION_STATS, [ + 'sessions/count', + 'sessions/max', + 'sessions/persistent/count', + 'sessions/persistent/max' +]). + +%% Subscribers, Subscriptions stats +-define(PUBSUB_STATS, [ + 'topics/count', + 'topics/max', + 'subscribers/count', + 'subscribers/max', + 'subscriptions/count', + 'subscriptions/max', + 'subscriptions/shared/count', + 'subscriptions/shared/max' +]). + +-define(ROUTE_STATS, [ + 'routes/count', + 'routes/max' +]). + +%% Retained stats +-define(RETAINED_STATS, [ + 'retained/count', + 'retained/max' +]). + +-define(TAB, ?MODULE). +-define(SERVER, ?MODULE). + +-type opts() :: #{tick_ms := timeout()}. + +%% @doc Start stats server +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + start_link(#{tick_ms => timer:seconds(1)}). + +-spec(start_link(opts()) -> emqx_types:startlink_ret()). +start_link(Opts) -> + gen_server:start_link({local, ?SERVER}, ?MODULE, Opts, []). + +-spec(stop() -> ok). +stop() -> + gen_server:call(?SERVER, stop, infinity). + +%% @doc Generate stats fun +-spec(statsfun(Stat :: atom()) -> fun()). +statsfun(Stat) -> + fun(Val) -> setstat(Stat, Val) end. + +-spec(statsfun(Stat :: atom(), MaxStat :: atom()) -> fun()). +statsfun(Stat, MaxStat) -> + fun(Val) -> setstat(Stat, MaxStat, Val) end. + +%% @doc Get all statistics +-spec(getstats() -> stats()). +getstats() -> + case ets:info(?TAB, name) of + undefined -> []; + _ -> ets:tab2list(?TAB) + end. + +%% @doc Get stats by name +-spec(getstat(atom()) -> non_neg_integer() | undefined). +getstat(Name) -> + case ets:lookup(?TAB, Name) of + [{Name, Val}] -> Val; + [] -> undefined + end. + +%% @doc Set stats +-spec(setstat(Stat :: atom(), Val :: pos_integer()) -> boolean()). +setstat(Stat, Val) when is_integer(Val) -> + safe_update_element(Stat, Val). + +%% @doc Set stats with max value. +-spec(setstat(Stat :: atom(), MaxStat :: atom(), + Val :: pos_integer()) -> boolean()). +setstat(Stat, MaxStat, Val) when is_integer(Val) -> + cast({setstat, Stat, MaxStat, Val}). + +-spec(update_interval(atom(), fun()) -> ok). +update_interval(Name, UpFun) -> + update_interval(Name, 1, UpFun). + +-spec(update_interval(atom(), pos_integer(), fun()) -> ok). +update_interval(Name, Secs, UpFun) when is_integer(Secs), Secs >= 1 -> + cast({update_interval, rec(Name, Secs, UpFun)}). + +-spec(cancel_update(atom()) -> ok). +cancel_update(Name) -> + cast({cancel_update, Name}). + +rec(Name, Secs, UpFun) -> + #update{name = Name, countdown = Secs, interval = Secs, func = UpFun}. + +cast(Msg) -> + gen_server:cast(?SERVER, Msg). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init(#{tick_ms := TickMs}) -> + ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]), + Stats = lists:append([?CONNECTION_STATS, ?SESSION_STATS, ?PUBSUB_STATS, + ?ROUTE_STATS, ?RETAINED_STATS]), + true = ets:insert(?TAB, [{Name, 0} || Name <- Stats]), + {ok, start_timer(#state{updates = [], tick_ms = TickMs}), hibernate}. + +start_timer(#state{tick_ms = Ms} = State) -> + State#state{timer = emqx_misc:start_timer(Ms, tick)}. + +handle_call(stop, _From, State) -> + {stop, normal, _Reply = ok, State}; +handle_call(Req, _From, State) -> + emqx_logger:error("[Stats] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({setstat, Stat, MaxStat, Val}, State) -> + try ets:lookup_element(?TAB, MaxStat, 2) of + MaxVal when Val > MaxVal -> + ets:update_element(?TAB, MaxStat, {2, Val}); + _ -> ok + catch + error:badarg -> + ets:insert(?TAB, {MaxStat, Val}) + end, + safe_update_element(Stat, Val), + {noreply, State}; + +handle_cast({update_interval, Update = #update{name = Name}}, State = #state{updates = Updates}) -> + case lists:keyfind(Name, #update.name, Updates) of + #update{} -> + emqx_logger:error("[Stats]: duplicated update: ~s", [Name]), + {noreply, State}; + false -> + {noreply, State#state{updates = [Update | Updates]}} + end; + +handle_cast({cancel_update, Name}, State = #state{updates = Updates}) -> + {noreply, State#state{updates = lists:keydelete(Name, #update.name, Updates)}}; + +handle_cast(Msg, State) -> + emqx_logger:error("[Stats] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({timeout, TRef, tick}, State = #state{timer = TRef, updates = Updates}) -> + Updates1 = lists:foldl( + fun(Update = #update{name = Name, countdown = C, interval = I, + func = UpFun}, Acc) when C =< 0 -> + try UpFun() + catch _:Error -> + emqx_logger:error("[Stats] update ~s error: ~p", [Name, Error]) + end, + [Update#update{countdown = I} | Acc]; + (Update = #update{countdown = C}, Acc) -> + [Update#update{countdown = C - 1} | Acc] + end, [], Updates), + {noreply, start_timer(State#state{updates = Updates1}), hibernate}; + +handle_info(Info, State) -> + emqx_logger:error("[Stats] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #state{timer = TRef}) -> + emqx_misc:cancel_timer(TRef). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +safe_update_element(Key, Val) -> + try ets:update_element(?TAB, Key, {2, Val}) + catch + error:badarg -> + ets:insert_new(?TAB, {Key, Val}) + end. + diff --git a/src/emqx_sup.erl b/src/emqx_sup.erl new file mode 100644 index 000000000..60be4db87 --- /dev/null +++ b/src/emqx_sup.erl @@ -0,0 +1,95 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sup). + +-behaviour(supervisor). + +-export([start_link/0, start_child/1, start_child/2, stop_child/1]). + +-export([init/1]). + +-type(startchild_ret() :: {ok, supervisor:child()} + | {ok, supervisor:child(), term()} + | {error, term()}). + +-define(SUPERVISOR, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SUPERVISOR}, ?MODULE, []). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec(start_child(supervisor:child_spec()) -> startchild_ret()). +start_child(ChildSpec) when is_tuple(ChildSpec) -> + supervisor:start_child(?SUPERVISOR, ChildSpec). + +-spec(start_child(module(), worker | supervisor) -> startchild_ret()). +start_child(Mod, worker) -> + start_child(worker_spec(Mod)); +start_child(Mod, supervisor) -> + start_child(supervisor_spec(Mod)). + +-spec(stop_child(supervisor:child_id()) -> ok | {error, term()}). +stop_child(ChildId) -> + case supervisor:terminate_child(?SUPERVISOR, ChildId) of + ok -> supervisor:delete_child(?SUPERVISOR, ChildId); + Error -> Error + end. + +%%-------------------------------------------------------------------- +%% Supervisor callbacks +%%-------------------------------------------------------------------- + +init([]) -> + %% Kernel Sup + KernelSup = supervisor_spec(emqx_kernel_sup), + %% Router Sup + RouterSup = supervisor_spec(emqx_router_sup), + %% Broker Sup + BrokerSup = supervisor_spec(emqx_broker_sup), + %% BridgeSup + LocalBridgeSup = supervisor_spec(emqx_local_bridge_sup_sup), + + BridgeSup = supervisor_spec(emqx_bridge_sup), + %% AccessControl + AccessControl = worker_spec(emqx_access_control), + %% Session Manager + SMSup = supervisor_spec(emqx_sm_sup), + %% Connection Manager + CMSup = supervisor_spec(emqx_cm_sup), + %% Sys Sup + SysSup = supervisor_spec(emqx_sys_sup), + {ok, {{one_for_all, 0, 1}, + [KernelSup, + RouterSup, + BrokerSup, + LocalBridgeSup, + BridgeSup, + AccessControl, + SMSup, + CMSup, + SysSup]}}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +worker_spec(M) -> + {M, {M, start_link, []}, permanent, 30000, worker, [M]}. +supervisor_spec(M) -> + {M, {M, start_link, []}, permanent, infinity, supervisor, [M]}. + diff --git a/src/emqx_sys.erl b/src/emqx_sys.erl new file mode 100644 index 000000000..5001f733b --- /dev/null +++ b/src/emqx_sys.erl @@ -0,0 +1,178 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sys). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/0]). +-export([version/0, uptime/0, datetime/0, sysdescr/0, sys_interval/0]). +-export([info/0]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-import(emqx_topic, [systop/1]). +-import(emqx_misc, [start_timer/2]). + +-record(state, {start_time, heartbeat, ticker, version, sysdescr}). + +-define(APP, emqx). +-define(SYS, ?MODULE). + +-define(INFO_KEYS, [ + version, % Broker version + uptime, % Broker uptime + datetime, % Broker local datetime + sysdescr % Broker description +]). + +-spec(start_link() -> {ok, pid()} | ignore | {error, any()}). +start_link() -> + gen_server:start_link({local, ?SYS}, ?MODULE, [], []). + +%% @doc Get sys version +-spec(version() -> string()). +version() -> + {ok, Version} = application:get_key(?APP, vsn), Version. + +%% @doc Get sys description +-spec(sysdescr() -> string()). +sysdescr() -> + {ok, Descr} = application:get_key(?APP, description), Descr. + +%% @doc Get sys uptime +-spec(uptime() -> string()). +uptime() -> + gen_server:call(?SYS, uptime). + +%% @doc Get sys datetime +-spec(datetime() -> string()). +datetime() -> + {{Y, M, D}, {H, MM, S}} = calendar:local_time(), + lists:flatten( + io_lib:format( + "~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S])). + +%% @doc Get sys interval +-spec(sys_interval() -> pos_integer()). +sys_interval() -> + application:get_env(?APP, sys_interval, 60000). + +%% @doc Get sys info +-spec(info() -> list(tuple())). +info() -> + [{version, version()}, + {sysdescr, sysdescr()}, + {uptime, uptime()}, + {datetime, datetime()}]. + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + State = #state{start_time = erlang:timestamp(), + version = iolist_to_binary(version()), + sysdescr = iolist_to_binary(sysdescr())}, + {ok, heartbeat(tick(State))}. + +heartbeat(State) -> + State#state{heartbeat = start_timer(timer:seconds(1), heartbeat)}. +tick(State) -> + State#state{ticker = start_timer(sys_interval(), tick)}. + +handle_call(uptime, _From, State) -> + {reply, uptime(State), State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[SYS] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[SYS] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({timeout, TRef, heartbeat}, State = #state{heartbeat = TRef}) -> + publish(uptime, iolist_to_binary(uptime(State))), + publish(datetime, iolist_to_binary(datetime())), + {noreply, heartbeat(State)}; + +handle_info({timeout, TRef, tick}, State = #state{ticker = TRef, version = Version, sysdescr = Descr}) -> + publish(version, Version), + publish(sysdescr, Descr), + publish(brokers, ekka_mnesia:running_nodes()), + publish(stats, emqx_stats:getstats()), + publish(metrics, emqx_metrics:all()), + {noreply, tick(State), hibernate}; + +handle_info(Info, State) -> + emqx_logger:error("[SYS] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #state{heartbeat = TRef1, ticker = TRef2}) -> + lists:foreach(fun emqx_misc:cancel_timer/1, [TRef1, TRef2]). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%----------------------------------------------------------------------------- +%% Internal functions +%%----------------------------------------------------------------------------- + +uptime(#state{start_time = Ts}) -> + Secs = timer:now_diff(erlang:timestamp(), Ts) div 1000000, + lists:flatten(uptime(seconds, Secs)). +uptime(seconds, Secs) when Secs < 60 -> + [integer_to_list(Secs), " seconds"]; +uptime(seconds, Secs) -> + [uptime(minutes, Secs div 60), integer_to_list(Secs rem 60), " seconds"]; +uptime(minutes, M) when M < 60 -> + [integer_to_list(M), " minutes, "]; +uptime(minutes, M) -> + [uptime(hours, M div 60), integer_to_list(M rem 60), " minutes, "]; +uptime(hours, H) when H < 24 -> + [integer_to_list(H), " hours, "]; +uptime(hours, H) -> + [uptime(days, H div 24), integer_to_list(H rem 24), " hours, "]; +uptime(days, D) -> + [integer_to_list(D), " days,"]. + +publish(uptime, Uptime) -> + safe_publish(systop(uptime), Uptime); +publish(datetime, Datetime) -> + safe_publish(systop(datatype), Datetime); +publish(version, Version) -> + safe_publish(systop(version), #{retain => true}, Version); +publish(sysdescr, Descr) -> + safe_publish(systop(sysdescr), #{retain => true}, Descr); +publish(brokers, Nodes) -> + Payload = string:join([atom_to_list(N) || N <- Nodes], ","), + safe_publish(<<"$SYS/brokers">>, #{retain => true}, Payload); +publish(stats, Stats) -> + [safe_publish(systop(lists:concat(['stats/', Stat])), integer_to_binary(Val)) + || {Stat, Val} <- Stats, is_atom(Stat), is_integer(Val)]; +publish(metrics, Metrics) -> + [safe_publish(systop(lists:concat(['metrics/', Metric])), integer_to_binary(Val)) + || {Metric, Val} <- Metrics, is_atom(Metric), is_integer(Val)]. + +safe_publish(Topic, Payload) -> + safe_publish(Topic, #{}, Payload). +safe_publish(Topic, Flags, Payload) -> + emqx_broker:safe_publish( + emqx_message:set_flags( + maps:merge(#{sys => true}, Flags), + emqx_message:make(?SYS, Topic, iolist_to_binary(Payload)))). + diff --git a/src/emqx_sys_mon.erl b/src/emqx_sys_mon.erl new file mode 100644 index 000000000..e3bd4a2a9 --- /dev/null +++ b/src/emqx_sys_mon.erl @@ -0,0 +1,162 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sys_mon). + +-behavior(gen_server). + +-export([start_link/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-record(state, {timer, events}). + +-define(SYSMON, ?MODULE). +-define(LOG(Msg, ProcInfo), + emqx_logger:warning(#{sysmon => true}, "[SYSMON] ~s~n~p", [WarnMsg, ProcInfo])). +-define(LOG(Msg, ProcInfo, PortInfo), + emqx_logger:warning(#{sysmon => true}, "[SYSMON] ~s~n~p~n~p", [WarnMsg, ProcInfo, PortInfo])). + +%% @doc Start system monitor +-spec(start_link(Opts :: list(tuple())) -> {ok, pid()} | ignore | {error, term()}). +start_link(Opts) -> + gen_server:start_link({local, ?SYSMON}, ?MODULE, [Opts], []). + +%%----------------------------------------------------------------------------- +%% gen_server callbacks +%%----------------------------------------------------------------------------- + +init([Opts]) -> + erlang:system_monitor(self(), parse_opt(Opts)), + {ok, start_timer(#state{events = []})}. + +start_timer(State) -> + State#state{timer = emqx_misc:start_timer(timer:seconds(2), reset)}. + +parse_opt(Opts) -> + parse_opt(Opts, []). +parse_opt([], Acc) -> + Acc; +parse_opt([{long_gc, false}|Opts], Acc) -> + parse_opt(Opts, Acc); +parse_opt([{long_gc, Ms}|Opts], Acc) when is_integer(Ms) -> + parse_opt(Opts, [{long_gc, Ms}|Acc]); +parse_opt([{long_schedule, false}|Opts], Acc) -> + parse_opt(Opts, Acc); +parse_opt([{long_schedule, Ms}|Opts], Acc) when is_integer(Ms) -> + parse_opt(Opts, [{long_schedule, Ms}|Acc]); +parse_opt([{large_heap, Size}|Opts], Acc) when is_integer(Size) -> + parse_opt(Opts, [{large_heap, Size}|Acc]); +parse_opt([{busy_port, true}|Opts], Acc) -> + parse_opt(Opts, [busy_port|Acc]); +parse_opt([{busy_port, false}|Opts], Acc) -> + parse_opt(Opts, Acc); +parse_opt([{busy_dist_port, true}|Opts], Acc) -> + parse_opt(Opts, [busy_dist_port|Acc]); +parse_opt([{busy_dist_port, false}|Opts], Acc) -> + parse_opt(Opts, Acc); +parse_opt([_Opt|Opts], Acc) -> + parse_opt(Opts, Acc). + +handle_call(Req, _From, State) -> + emqx_logger:error("[SYSMON] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[SYSMON] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({monitor, Pid, long_gc, Info}, State) -> + suppress({long_gc, Pid}, + fun() -> + WarnMsg = io_lib:format("long_gc warning: pid = ~p, info: ~p", [Pid, Info]), + ?LOG(WarnMsg, procinfo(Pid)), + safe_publish(long_gc, WarnMsg) + end, State); + +handle_info({monitor, Pid, long_schedule, Info}, State) when is_pid(Pid) -> + suppress({long_schedule, Pid}, + fun() -> + WarnMsg = io_lib:format("long_schedule warning: pid = ~p, info: ~p", [Pid, Info]), + ?LOG(WarnMsg, procinfo(Pid)), + safe_publish(long_schedule, WarnMsg) + end, State); + +handle_info({monitor, Port, long_schedule, Info}, State) when is_port(Port) -> + suppress({long_schedule, Port}, + fun() -> + WarnMsg = io_lib:format("long_schedule warning: port = ~p, info: ~p", [Port, Info]), + ?LOG(WarnMsg, erlang:port_info(Port)), + safe_publish(long_schedule, WarnMsg) + end, State); + +handle_info({monitor, Pid, large_heap, Info}, State) -> + suppress({large_heap, Pid}, + fun() -> + WarnMsg = io_lib:format("large_heap warning: pid = ~p, info: ~p", [Pid, Info]), + ?LOG(WarnMsg, procinfo(Pid)), + safe_publish(large_heap, WarnMsg) + end, State); + +handle_info({monitor, SusPid, busy_port, Port}, State) -> + suppress({busy_port, Port}, + fun() -> + WarnMsg = io_lib:format("busy_port warning: suspid = ~p, port = ~p", [SusPid, Port]), + ?LOG(WarnMsg, procinfo(SusPid), erlang:port_info(Port)), + safe_publish(busy_port, WarnMsg) + end, State); + +handle_info({monitor, SusPid, busy_dist_port, Port}, State) -> + suppress({busy_dist_port, Port}, + fun() -> + WarnMsg = io_lib:format("busy_dist_port warning: suspid = ~p, port = ~p", [SusPid, Port]), + ?LOG(WarnMsg, procinfo(SusPid), erlang:port_info(Port)), + safe_publish(busy_dist_port, WarnMsg) + end, State); + +handle_info({timeout, _Ref, reset}, State) -> + {noreply, State#state{events = []}, hibernate}; + +handle_info(Info, State) -> + logger:error("[SYSMON] unexpected Info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, #state{timer = TRef}) -> + emqx_misc:cancel_timer(TRef). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +suppress(Key, SuccFun, State = #state{events = Events}) -> + case lists:member(Key, Events) of + true -> {noreply, State}; + false -> SuccFun(), + {noreply, State#state{events = [Key|Events]}} + end. + +procinfo(Pid) -> + case {emqx_vm:get_process_info(Pid), emqx_vm:get_process_gc(Pid)} of + {undefined, _} -> undefined; + {_, undefined} -> undefined; + {Info, GcInfo} -> Info ++ GcInfo + end. + +safe_publish(Event, WarnMsg) -> + Topic = emqx_topic:systop(lists:concat(['sysmon/', Event])), + emqx_broker:safe_publish(sysmon_msg(Topic, iolist_to_binary(WarnMsg))). + +sysmon_msg(Topic, Payload) -> + emqx_message:make(?SYSMON, #{sys => true}, Topic, Payload). + diff --git a/src/emqx_sys_sup.erl b/src/emqx_sys_sup.erl new file mode 100644 index 000000000..b450ddd66 --- /dev/null +++ b/src/emqx_sys_sup.erl @@ -0,0 +1,40 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sys_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Sys = #{id => sys, + start => {emqx_sys, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_sys]}, + Sysmon = #{id => sys_mon, + start => {emqx_sys_mon, start_link, [emqx_config:get_env(sysmon, [])]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_sys_mon]}, + {ok, {{one_for_one, 10, 100}, [Sys, Sysmon]}}. + diff --git a/src/emqx_tables.erl b/src/emqx_tables.erl new file mode 100644 index 000000000..fdb106a99 --- /dev/null +++ b/src/emqx_tables.erl @@ -0,0 +1,42 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_tables). + +-export([new/2]). +-export([lookup_value/2, lookup_value/3]). + +%% Create a named_table ets. +-spec(new(atom(), list()) -> ok). +new(Tab, Opts) -> + case ets:info(Tab, name) of + undefined -> + _ = ets:new(Tab, lists:usort([named_table | Opts])), + ok; + Tab -> ok + end. + +%% KV lookup +-spec(lookup_value(atom(), term()) -> any()). +lookup_value(Tab, Key) -> + lookup_value(Tab, Key, undefined). + +-spec(lookup_value(atom(), term(), any()) -> any()). +lookup_value(Tab, Key, Def) -> + try + ets:lookup_element(Tab, Key, 2) + catch + error:badarg -> Def + end. + diff --git a/src/emqttd_time.erl b/src/emqx_time.erl similarity index 58% rename from src/emqttd_time.erl rename to src/emqx_time.erl index 77459195e..95bfc9409 100644 --- a/src/emqttd_time.erl +++ b/src/emqx_time.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,29 +11,22 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_time). +-module(emqx_time). --author("Feng Lee "). - --export([seed/0, now_secs/0, now_secs/1, now_ms/0, now_ms/1, ts_from_ms/1]). +-export([seed/0, now_secs/0, now_secs/1, now_ms/0, now_ms/1]). seed() -> rand:seed(exsplus, erlang:timestamp()). -now_ms() -> - now_ms(os:timestamp()). - -now_ms({MegaSecs, Secs, MicroSecs}) -> - (MegaSecs * 1000000 + Secs) * 1000 + round(MicroSecs/1000). - now_secs() -> - now_secs(os:timestamp()). + erlang:system_time(second). now_secs({MegaSecs, Secs, _MicroSecs}) -> MegaSecs * 1000000 + Secs. -ts_from_ms(Ms) -> - {Ms div 1000000, Ms rem 1000000, 0}. +now_ms() -> + erlang:system_time(millisecond). +now_ms({MegaSecs, Secs, MicroSecs}) -> + (MegaSecs * 1000000 + Secs) * 1000 + round(MicroSecs/1000). diff --git a/src/emqttd_topic.erl b/src/emqx_topic.erl similarity index 60% rename from src/emqttd_topic.erl rename to src/emqx_topic.erl index 6623c730f..f7e229a13 100644 --- a/src/emqttd_topic.erl +++ b/src/emqx_topic.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,43 +11,36 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_topic). - --author("Feng Lee "). - --include("emqttd_protocol.hrl"). - --include("emqttd_internal.hrl"). - --import(lists, [reverse/1]). - --export([match/2, validate/1, triples/1, words/1, wildcard/1]). - --export([join/1, feed_var/3, systop/1]). +-module(emqx_topic). +-export([match/2]). +-export([validate/1, validate/2]). +-export([levels/1]). +-export([triples/1]). +-export([words/1]). +-export([wildcard/1]). +-export([join/1]). +-export([feed_var/3]). +-export([systop/1]). -export([parse/1, parse/2]). -type(topic() :: binary()). - --type(option() :: local | {qos, mqtt_qos()} | {share, '$queue' | binary()}). - --type(word() :: '' | '+' | '#' | binary()). - --type(words() :: list(word())). - +-type(word() :: '' | '+' | '#' | binary()). +-type(words() :: list(word())). -type(triple() :: {root | binary(), word(), binary()}). --export_type([topic/0, option/0, word/0, triple/0]). +-export_type([topic/0, word/0, triple/0]). -define(MAX_TOPIC_LEN, 4096). +-include("emqx_mqtt.hrl"). + %% @doc Is wildcard topic? -spec(wildcard(topic() | words()) -> true | false). wildcard(Topic) when is_binary(Topic) -> wildcard(words(Topic)); -wildcard([]) -> +wildcard([]) -> false; wildcard(['#'|_]) -> true; @@ -82,15 +74,21 @@ match([_H1|_], []) -> match([], [_H|_T2]) -> false. -%% @doc Validate Topic --spec(validate({name | filter, topic()}) -> boolean()). -validate({_, <<>>}) -> - false; -validate({_, Topic}) when is_binary(Topic) and (size(Topic) > ?MAX_TOPIC_LEN) -> - false; -validate({filter, Topic}) when is_binary(Topic) -> +%% @doc Validate topic name or filter +-spec(validate(topic() | {name | filter, topic()}) -> true). +validate(Topic) when is_binary(Topic) -> + validate(filter, Topic); +validate({Type, Topic}) when Type =:= name; Type =:= filter -> + validate(Type, Topic). + +-spec(validate(name | filter, topic()) -> true). +validate(_, <<>>) -> + error(empty_topic); +validate(_, Topic) when is_binary(Topic) and (size(Topic) > ?MAX_TOPIC_LEN) -> + error(topic_too_long); +validate(filter, Topic) when is_binary(Topic) -> validate2(words(Topic)); -validate({name, Topic}) when is_binary(Topic) -> +validate(name, Topic) when is_binary(Topic) -> Words = words(Topic), validate2(Words) and (not wildcard(Words)). @@ -99,29 +97,28 @@ validate2([]) -> validate2(['#']) -> % end with '#' true; validate2(['#'|Words]) when length(Words) > 0 -> - false; + error('topic_invalid_#'); validate2([''|Words]) -> validate2(Words); validate2(['+'|Words]) -> validate2(Words); validate2([W|Words]) -> - case validate3(W) of true -> validate2(Words); false -> false end. + validate3(W) andalso validate2(Words). validate3(<<>>) -> true; validate3(<>) when C == $#; C == $+; C == 0 -> - false; + error('topic_invalid_char'); validate3(<<_/utf8, Rest/binary>>) -> validate3(Rest). -%% @doc Topic to Triples +%% @doc Topic to triples -spec(triples(topic()) -> list(triple())). triples(Topic) when is_binary(Topic) -> triples(words(Topic), root, []). triples([], _Parent, Acc) -> - reverse(Acc); - + lists:reverse(Acc); triples([W|Words], Parent, Acc) -> Node = join(Parent, W), triples(Words, Node, [{Parent, W, Node}|Acc]). @@ -136,6 +133,9 @@ bin('+') -> <<"+">>; bin('#') -> <<"#">>; bin(B) when is_binary(B) -> B. +levels(Topic) when is_binary(Topic) -> + length(words(Topic)). + %% @doc Split Topic Path to Words -spec(words(topic()) -> words()). words(Topic) when is_binary(Topic) -> @@ -147,17 +147,16 @@ word(<<"#">>) -> '#'; word(Bin) -> Bin. %% @doc '$SYS' Topic. -systop(Name) when is_atom(Name) -> - list_to_binary(lists:concat(["$SYS/brokers/", node(), "/", Name])); - +systop(Name) when is_atom(Name); is_list(Name) -> + iolist_to_binary(lists:concat(["$SYS/brokers/", node(), "/", Name])); systop(Name) when is_binary(Name) -> - list_to_binary(["$SYS/brokers/", atom_to_list(node()), "/", Name]). + iolist_to_binary(["$SYS/brokers/", atom_to_list(node()), "/", Name]). -spec(feed_var(binary(), binary(), binary()) -> binary()). feed_var(Var, Val, Topic) -> feed_var(Var, Val, words(Topic), []). feed_var(_Var, _Val, [], Acc) -> - join(reverse(Acc)); + join(lists:reverse(Acc)); feed_var(Var, Val, [Var|Words], Acc) -> feed_var(Var, Val, Words, [Val|Acc]); feed_var(Var, Val, [W|Words], Acc) -> @@ -171,38 +170,32 @@ join([W]) -> join(Words) -> {_, Bin} = lists:foldr(fun(W, {true, Tail}) -> - {false, <>}; + {false, <>}; (W, {false, Tail}) -> - {false, <>} + {false, <>} end, {true, <<>>}, [bin(W) || W <- Words]), Bin. --spec(parse(topic()) -> {topic(), [option()]}). +-spec(parse(topic()) -> {topic(), #{}}). parse(Topic) when is_binary(Topic) -> - parse(Topic, []). - -parse(<<"$local/", Topic1/binary>>, Options) -> - if_not_contain(local, Options, fun() -> - parse(Topic1, [local | Options]) - end); + parse(Topic, #{}). +parse(Topic = <<"$queue/", _/binary>>, #{share := _Group}) -> + error({invalid_topic, Topic}); +parse(Topic = <>, #{share := _Group}) -> + error({invalid_topic, Topic}); parse(<<"$queue/", Topic1/binary>>, Options) -> - if_not_contain(share, Options,fun() -> - parse(Topic1, [{share, '$queue'} | Options]) - end); - -parse(<<"$share/", Topic1/binary>>, Options) -> - if_not_contain(share, Options, fun() -> - [Share, Topic2] = binary:split(Topic1, <<"/">>), - {Topic2, [{share, Share} | Options]} - end); - + parse(Topic1, maps:put(share, <<"$queue">>, Options)); +parse(Topic = <>, Options) -> + case binary:split(Topic1, <<"/">>) of + [<<>>] -> error({invalid_topic, Topic}); + [_] -> error({invalid_topic, Topic}); + [Group, Topic2] -> + case binary:match(Group, [<<"/">>, <<"+">>, <<"#">>]) of + nomatch -> {Topic2, maps:put(share, Group, Options)}; + _ -> error({invalid_topic, Topic}) + end + end; parse(Topic, Options) -> {Topic, Options}. -if_not_contain(local, Options, Fun) -> - ?IF(lists:member(local, Options), error(invalid_topic), Fun()); - -if_not_contain(share, Options, Fun) -> - ?IF(lists:keyfind(share, 1, Options), error(invalid_topic), Fun()). - diff --git a/src/emqx_tracer.erl b/src/emqx_tracer.erl new file mode 100644 index 000000000..d92cb4a1f --- /dev/null +++ b/src/emqx_tracer.erl @@ -0,0 +1,154 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_tracer). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/0]). +-export([trace/2]). +-export([start_trace/3, lookup_traces/0, stop_trace/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-record(state, {traces}). + +-type(trace_who() :: {client_id | topic, binary()}). + +-define(TRACER, ?MODULE). +-define(FORMAT, {emqx_logger_formatter, + #{template => + [time," [",level,"] ", + {client_id, + [{peername, + [client_id,"@",peername," "], + [client_id, " "]}], + [{peername, + [peername," "], + []}]}, + msg,"\n"]}}). + +-spec(start_link() -> {ok, pid()} | ignore | {error, term()}). +start_link() -> + gen_server:start_link({local, ?TRACER}, ?MODULE, [], []). + +trace(publish, #message{topic = <<"$SYS/", _/binary>>}) -> + %% Dont' trace '$SYS' publish + ignore; +trace(publish, #message{from = From, topic = Topic, payload = Payload}) + when is_binary(From); is_atom(From) -> + emqx_logger:info(#{topic => Topic}, "PUBLISH to ~s: ~p", [Topic, Payload]). + +%%------------------------------------------------------------------------------ +%% Start/Stop trace +%%------------------------------------------------------------------------------ + +%% @doc Start to trace client_id or topic. +-spec(start_trace(trace_who(), logger:level(), string()) -> ok | {error, term()}). +start_trace({client_id, ClientId}, Level, LogFile) -> + start_trace({start_trace, {client_id, ClientId}, Level, LogFile}); +start_trace({topic, Topic}, Level, LogFile) -> + start_trace({start_trace, {topic, Topic}, Level, LogFile}). + +start_trace(Req) -> gen_server:call(?MODULE, Req, infinity). + +%% @doc Stop tracing client_id or topic. +-spec(stop_trace(trace_who()) -> ok | {error, term()}). +stop_trace({client_id, ClientId}) -> + gen_server:call(?MODULE, {stop_trace, {client_id, ClientId}}); +stop_trace({topic, Topic}) -> + gen_server:call(?MODULE, {stop_trace, {topic, Topic}}). + +%% @doc Lookup all traces +-spec(lookup_traces() -> [{Who :: trace_who(), LogFile :: string()}]). +lookup_traces() -> + gen_server:call(?TRACER, lookup_traces). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + {ok, #state{traces = #{}}}. + +handle_call({start_trace, Who, Level, LogFile}, _From, State = #state{traces = Traces}) -> + case logger:add_handler(handler_id(Who), logger_disk_log_h, + #{level => Level, + formatter => ?FORMAT, + filesync_repeat_interval => no_repeat, + config => #{type => halt, file => LogFile}, + filter_default => stop, + filters => [{meta_key_filter, + {fun filter_by_meta_key/2, Who} }]}) of + ok -> + emqx_logger:info("[Tracer] start trace for ~p", [Who]), + {reply, ok, State#state{traces = maps:put(Who, {Level, LogFile}, Traces)}}; + {error, Reason} -> + emqx_logger:error("[Tracer] start trace for ~p failed, error: ~p", [Who, Reason]), + {reply, {error, Reason}, State} + end; + +handle_call({stop_trace, Who}, _From, State = #state{traces = Traces}) -> + case maps:find(Who, Traces) of + {ok, _LogFile} -> + case logger:remove_handler(handler_id(Who)) of + ok -> + emqx_logger:info("[Tracer] stop trace for ~p", [Who]); + {error, Reason} -> + emqx_logger:error("[Tracer] stop trace for ~p failed, error: ~p", [Who, Reason]) + end, + {reply, ok, State#state{traces = maps:remove(Who, Traces)}}; + error -> + {reply, {error, not_found}, State} + end; + +handle_call(lookup_traces, _From, State = #state{traces = Traces}) -> + {reply, [{Who, LogFile} || {Who, LogFile} <- maps:to_list(Traces)], State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[Tracer] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + emqx_logger:error("[Tracer] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(Info, State) -> + emqx_logger:error("[Tracer] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +handler_id({topic, Topic}) -> + list_to_atom("topic_" ++ binary_to_list(Topic)); +handler_id({client_id, ClientId}) -> + list_to_atom("clientid_" ++ binary_to_list(ClientId)). + +filter_by_meta_key(#{meta:=Meta}=LogEvent, {MetaKey, MetaValue}) -> + case maps:find(MetaKey, Meta) of + {ok, MetaValue} -> LogEvent; + {ok, Topic} when MetaKey =:= topic -> + case emqx_topic:match(Topic, MetaValue) of + true -> LogEvent; + false -> ignore + end; + _ -> ignore + end. \ No newline at end of file diff --git a/src/emqttd_trie.erl b/src/emqx_trie.erl similarity index 54% rename from src/emqttd_trie.erl rename to src/emqx_trie.erl index 2dae6974a..27ff52827 100644 --- a/src/emqttd_trie.erl +++ b/src/emqx_trie.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,130 +11,130 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- -%% @doc MQTT Topic Trie: -%% [Trie](http://en.wikipedia.org/wiki/Trie) -%% @end +-module(emqx_trie). --module(emqttd_trie). +-include("emqx.hrl"). --author("Feng Lee "). - --include("emqttd_trie.hrl"). - -%% Mnesia Callbacks +%% Mnesia bootstrap -export([mnesia/1]). -boot_mnesia({mnesia, [boot]}). -copy_mnesia({mnesia, [copy]}). -%% Trie API +%% Trie APIs -export([insert/1, match/1, lookup/1, delete/1]). -%%-------------------------------------------------------------------- -%% Mnesia Callbacks -%%-------------------------------------------------------------------- +%% Mnesia tables +-define(TRIE, emqx_trie). +-define(TRIE_NODE, emqx_trie_node). -%% @doc Create or Replicate trie tables. +%%------------------------------------------------------------------------------ +%% Mnesia bootstrap +%%------------------------------------------------------------------------------ + +%% @doc Create or replicate trie tables. -spec(mnesia(boot | copy) -> ok). mnesia(boot) -> - %% Trie Table - ok = ekka_mnesia:create_table(mqtt_trie, [ + %% Optimize storage + StoreProps = [{ets, [{read_concurrency, true}, + {write_concurrency, true}]}], + %% Trie table + ok = ekka_mnesia:create_table(?TRIE, [ {ram_copies, [node()]}, {record_name, trie}, - {attributes, record_info(fields, trie)}]), - %% Trie Node Table - ok = ekka_mnesia:create_table(mqtt_trie_node, [ + {attributes, record_info(fields, trie)}, + {storage_properties, StoreProps}]), + %% Trie node table + ok = ekka_mnesia:create_table(?TRIE_NODE, [ {ram_copies, [node()]}, {record_name, trie_node}, - {attributes, record_info(fields, trie_node)}]); + {attributes, record_info(fields, trie_node)}, + {storage_properties, StoreProps}]); mnesia(copy) -> - %% Copy Trie Table - ok = ekka_mnesia:copy_table(mqtt_trie), - %% Copy Trie Node Table - ok = ekka_mnesia:copy_table(mqtt_trie_node). + %% Copy trie table + ok = ekka_mnesia:copy_table(?TRIE), + %% Copy trie_node table + ok = ekka_mnesia:copy_table(?TRIE_NODE). -%%-------------------------------------------------------------------- -%% Trie API -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ +%% Trie APIs +%%------------------------------------------------------------------------------ -%% @doc Insert topic to trie --spec(insert(Topic :: binary()) -> ok). +%% @doc Insert a topic filter into the trie. +-spec(insert(emqx_topic:topic()) -> ok). insert(Topic) when is_binary(Topic) -> - case mnesia:read(mqtt_trie_node, Topic) of + case mnesia:wread({?TRIE_NODE, Topic}) of [#trie_node{topic = Topic}] -> ok; [TrieNode = #trie_node{topic = undefined}] -> write_trie_node(TrieNode#trie_node{topic = Topic}); [] -> - % Add trie path - lists:foreach(fun add_path/1, emqttd_topic:triples(Topic)), - % Add last node + %% Add trie path + ok = lists:foreach(fun add_path/1, emqx_topic:triples(Topic)), + %% Add last node write_trie_node(#trie_node{node_id = Topic, topic = Topic}) end. -%% @doc Find trie nodes that match topic --spec(match(Topic :: binary()) -> list(MatchedTopic :: binary())). +%% @doc Find trie nodes that match the topic name. +-spec(match(emqx_topic:topic()) -> list(emqx_topic:topic())). match(Topic) when is_binary(Topic) -> - TrieNodes = match_node(root, emqttd_topic:words(Topic)), + TrieNodes = match_node(root, emqx_topic:words(Topic)), [Name || #trie_node{topic = Name} <- TrieNodes, Name =/= undefined]. -%% @doc Lookup a Trie Node +%% @doc Lookup a trie node. -spec(lookup(NodeId :: binary()) -> [#trie_node{}]). lookup(NodeId) -> - mnesia:read(mqtt_trie_node, NodeId). + mnesia:read(?TRIE_NODE, NodeId). -%% @doc Delete topic from trie --spec(delete(Topic :: binary()) -> ok). +%% @doc Delete a topic filter from the trie. +-spec(delete(emqx_topic:topic()) -> ok). delete(Topic) when is_binary(Topic) -> - case mnesia:read(mqtt_trie_node, Topic) of + case mnesia:wread({?TRIE_NODE, Topic}) of [#trie_node{edge_count = 0}] -> - mnesia:delete({mqtt_trie_node, Topic}), - delete_path(lists:reverse(emqttd_topic:triples(Topic))); + ok = mnesia:delete({?TRIE_NODE, Topic}), + delete_path(lists:reverse(emqx_topic:triples(Topic))); [TrieNode] -> write_trie_node(TrieNode#trie_node{topic = undefined}); - [] -> - ok + [] -> ok end. -%%-------------------------------------------------------------------- -%% Internal Functions -%%-------------------------------------------------------------------- +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ %% @private -%% @doc Add path to trie tree. +%% @doc Add a path to the trie. add_path({Node, Word, Child}) -> Edge = #trie_edge{node_id = Node, word = Word}, - case mnesia:read(mqtt_trie_node, Node) of + case mnesia:wread({?TRIE_NODE, Node}) of [TrieNode = #trie_node{edge_count = Count}] -> - case mnesia:wread({mqtt_trie, Edge}) of + case mnesia:wread({?TRIE, Edge}) of [] -> - write_trie_node(TrieNode#trie_node{edge_count = Count+1}), + ok = write_trie_node(TrieNode#trie_node{edge_count = Count + 1}), write_trie(#trie{edge = Edge, node_id = Child}); - [_] -> - ok + [_] -> ok end; [] -> - write_trie_node(#trie_node{node_id = Node, edge_count = 1}), + ok = write_trie_node(#trie_node{node_id = Node, edge_count = 1}), write_trie(#trie{edge = Edge, node_id = Child}) end. %% @private %% @doc Match node with word or '+'. -match_node(root, [<<"$SYS">>|Words]) -> - match_node(<<"$SYS">>, Words, []); +match_node(root, [NodeId = <<$$, _/binary>>|Words]) -> + match_node(NodeId, Words, []); match_node(NodeId, Words) -> match_node(NodeId, Words, []). match_node(NodeId, [], ResAcc) -> - mnesia:read(mqtt_trie_node, NodeId) ++ 'match_#'(NodeId, ResAcc); + mnesia:read(?TRIE_NODE, NodeId) ++ 'match_#'(NodeId, ResAcc); match_node(NodeId, [W|Words], ResAcc) -> lists:foldl(fun(WArg, Acc) -> - case mnesia:read(mqtt_trie, #trie_edge{node_id = NodeId, word = WArg}) of + case mnesia:read(?TRIE, #trie_edge{node_id = NodeId, word = WArg}) of [#trie{node_id = ChildId}] -> match_node(ChildId, Words, Acc); [] -> Acc end @@ -144,22 +143,21 @@ match_node(NodeId, [W|Words], ResAcc) -> %% @private %% @doc Match node with '#'. 'match_#'(NodeId, ResAcc) -> - case mnesia:read(mqtt_trie, #trie_edge{node_id = NodeId, word = '#'}) of + case mnesia:read(?TRIE, #trie_edge{node_id = NodeId, word = '#'}) of [#trie{node_id = ChildId}] -> - mnesia:read(mqtt_trie_node, ChildId) ++ ResAcc; - [] -> - ResAcc + mnesia:read(?TRIE_NODE, ChildId) ++ ResAcc; + [] -> ResAcc end. %% @private -%% @doc Delete paths from trie tree. +%% @doc Delete paths from the trie. delete_path([]) -> ok; delete_path([{NodeId, Word, _} | RestPath]) -> - mnesia:delete({mqtt_trie, #trie_edge{node_id = NodeId, word = Word}}), - case mnesia:read(mqtt_trie_node, NodeId) of + ok = mnesia:delete({?TRIE, #trie_edge{node_id = NodeId, word = Word}}), + case mnesia:wread({?TRIE_NODE, NodeId}) of [#trie_node{edge_count = 1, topic = undefined}] -> - mnesia:delete({mqtt_trie_node, NodeId}), + ok = mnesia:delete({?TRIE_NODE, NodeId}), delete_path(RestPath); [TrieNode = #trie_node{edge_count = 1, topic = _}] -> write_trie_node(TrieNode#trie_node{edge_count = 0}); @@ -171,9 +169,9 @@ delete_path([{NodeId, Word, _} | RestPath]) -> %% @private write_trie(Trie) -> - mnesia:write(mqtt_trie, Trie, write). + mnesia:write(?TRIE, Trie, write). %% @private write_trie_node(TrieNode) -> - mnesia:write(mqtt_trie_node, TrieNode, write). + mnesia:write(?TRIE_NODE, TrieNode, write). diff --git a/src/emqx_types.erl b/src/emqx_types.erl new file mode 100644 index 000000000..960aa699a --- /dev/null +++ b/src/emqx_types.erl @@ -0,0 +1,64 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_types). + +-include("emqx.hrl"). + +-export_type([zone/0]). +-export_type([startlink_ret/0, ok_or_error/1]). +-export_type([pubsub/0, topic/0, subid/0, subopts/0]). +-export_type([client_id/0, username/0, password/0, peername/0, protocol/0]). +-export_type([credentials/0, session/0]). +-export_type([subscription/0, subscriber/0, topic_table/0]). +-export_type([payload/0, message/0]). +-export_type([delivery/0, deliver_results/0]). +-export_type([route/0]). +-export_type([alarm/0, plugin/0, command/0]). + +-type(zone() :: atom()). +-type(startlink_ret() :: {ok, pid()} | ignore | {error, term()}). +-type(ok_or_error(Reason) :: ok | {error, Reason}). +-type(pubsub() :: publish | subscribe). +-type(topic() :: binary()). +-type(subid() :: binary() | atom()). +-type(subopts() :: #{qos := integer(), + share => binary(), + atom() => term() + }). +-type(session() :: #session{}). +-type(client_id() :: binary() | atom()). +-type(username() :: binary() | undefined). +-type(password() :: binary() | undefined). +-type(peername() :: {inet:ip_address(), inet:port_number()}). +-type(protocol() :: mqtt | 'mqtt-sn' | coap | stomp | none | atom()). +-type(credentials() :: #{client_id := client_id(), + username := username(), + peername := peername(), + zone => zone(), + atom() => term() + }). +-type(subscription() :: #subscription{}). +-type(subscriber() :: {pid(), subid()}). +-type(topic_table() :: [{topic(), subopts()}]). +-type(payload() :: binary() | iodata()). +-type(message() :: #message{}). +-type(delivery() :: #delivery{}). +-type(deliver_results() :: [{route, node(), topic()} | + {dispatch, topic(), pos_integer()}]). +-type(route() :: #route{}). +-type(alarm() :: #alarm{}). +-type(plugin() :: #plugin{}). +-type(command() :: #command{}). + diff --git a/src/emqttd_vm.erl b/src/emqx_vm.erl similarity index 97% rename from src/emqttd_vm.erl rename to src/emqx_vm.erl index 725d7c920..8da40cd99 100644 --- a/src/emqttd_vm.erl +++ b/src/emqx_vm.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,26 +11,19 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_vm). +-module(emqx_vm). -export([schedulers/0]). - -export([microsecs/0]). - -export([loads/0, get_system_info/0, get_system_info/1, mem_info/0, scheduler_usage/1]). - -export([get_memory/0]). - -export([get_process_list/0, get_process_info/0, get_process_info/1, get_process_gc/0, get_process_gc/1, get_process_group_leader_info/1, get_process_limit/0]). - -export([get_ets_list/0, get_ets_info/0, get_ets_info/1, get_ets_object/0, get_ets_object/1]). - -export([get_port_types/0, get_port_info/0, get_port_info/1]). -define(UTIL_ALLOCATORS, [temp_alloc, @@ -204,13 +196,13 @@ mem_info() -> [{total_memory, proplists:get_value(total_memory, Dataset)}, {used_memory, proplists:get_value(total_memory, Dataset) - proplists:get_value(free_memory, Dataset)}]. -ftos(F) -> - [S] = io_lib:format("~.2f", [F]), S. +ftos(F) -> + S = io_lib:format("~.2f", [F]), S. -%%%% erlang vm scheduler_usage fun copied from recon +%%%% erlang vm scheduler_usage fun copied from recon scheduler_usage(Interval) when is_integer(Interval) -> %% We start and stop the scheduler_wall_time system flag - %% if it wasn't in place already. Usually setting the flag + %% if it wasn't in place already. Usually setting the flag %% should have a CPU impact(make it higher) only when under low usage. FormerFlag = erlang:system_flag(scheduler_wall_time, true), First = erlang:statistics(scheduler_wall_time), @@ -300,7 +292,7 @@ get_process_group_leader_info(LeaderPid) when is_pid(LeaderPid) -> [{Key, Value}|| {Key, Value} <- process_info(LeaderPid), lists:member(Key, ?PROCESS_INFO)]. get_process_limit() -> - erlang:system_info(process_limit). + erlang:system_info(process_limit). get_ets_list() -> ets:all(). diff --git a/src/emqx_ws_connection.erl b/src/emqx_ws_connection.erl new file mode 100644 index 000000000..74ef32c0a --- /dev/null +++ b/src/emqx_ws_connection.erl @@ -0,0 +1,304 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_ws_connection). + +-define(LOG_HEADER, "[WS]"). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). +-include("logger.hrl"). + +-export([info/1, attrs/1]). +-export([stats/1]). +-export([kick/1]). +-export([session/1]). + +%% websocket callbacks +-export([init/2]). +-export([websocket_init/1]). +-export([websocket_handle/2]). +-export([websocket_info/2]). +-export([terminate/3]). + +-record(state, { + request, + options, + peername, + sockname, + idle_timeout, + proto_state, + parser_state, + keepalive, + enable_stats, + stats_timer, + shutdown + }). + +-define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%% for debug +info(WSPid) when is_pid(WSPid) -> + call(WSPid, info); + +info(#state{peername = Peername, + sockname = Sockname, + proto_state = ProtoState}) -> + ProtoInfo = emqx_protocol:info(ProtoState), + ConnInfo = [{socktype, websocket}, + {conn_state, running}, + {peername, Peername}, + {sockname, Sockname}], + lists:append([ConnInfo, ProtoInfo]). + +%% for dashboard +attrs(WSPid) when is_pid(WSPid) -> + call(WSPid, attrs); + +attrs(#state{peername = Peername, + sockname = Sockname, + proto_state = ProtoState}) -> + SockAttrs = [{peername, Peername}, + {sockname, Sockname}], + ProtoAttrs = emqx_protocol:attrs(ProtoState), + lists:usort(lists:append(SockAttrs, ProtoAttrs)). + +stats(WSPid) when is_pid(WSPid) -> + call(WSPid, stats); + +stats(#state{proto_state = ProtoState}) -> + lists:append([wsock_stats(), + emqx_misc:proc_stats(), + emqx_protocol:stats(ProtoState) + ]). + +kick(WSPid) when is_pid(WSPid) -> + call(WSPid, kick). + +session(WSPid) when is_pid(WSPid) -> + call(WSPid, session). + +call(WSPid, Req) when is_pid(WSPid) -> + Mref = erlang:monitor(process, WSPid), + WSPid ! {call, {self(), Mref}, Req}, + receive + {Mref, Reply} -> + erlang:demonitor(Mref, [flush]), + Reply; + {'DOWN', Mref, _, _, Reason} -> + exit(Reason) + after 5000 -> + erlang:demonitor(Mref, [flush]), + exit(timeout) + end. + +%%------------------------------------------------------------------------------ +%% WebSocket callbacks +%%------------------------------------------------------------------------------ + +init(Req, Opts) -> + case cowboy_req:parse_header(<<"sec-websocket-protocol">>, Req) of + undefined -> + {cowboy_websocket, Req, #state{}}; + [<<"mqtt", Vsn/binary>>] -> + Resp = cowboy_req:set_resp_header(<<"sec-websocket-protocol">>, <<"mqtt", Vsn/binary>>, Req), + {cowboy_websocket, Resp, #state{request = Req, options = Opts}, #{idle_timeout => 86400000}}; + _ -> + {ok, cowboy_req:reply(400, Req), #state{}} + end. + +websocket_init(#state{request = Req, options = Options}) -> + Peername = cowboy_req:peer(Req), + Sockname = cowboy_req:sock(Req), + Peercert = cowboy_req:cert(Req), + ProtoState = emqx_protocol:init(#{peername => Peername, + sockname => Sockname, + peercert => Peercert, + sendfun => send_fun(self())}, Options), + ParserState = emqx_protocol:parser(ProtoState), + Zone = proplists:get_value(zone, Options), + EnableStats = emqx_zone:get_env(Zone, enable_stats, true), + IdleTimout = emqx_zone:get_env(Zone, idle_timeout, 30000), + + emqx_logger:set_metadata_peername(esockd_net:format(Peername)), + {ok, #state{peername = Peername, + sockname = Sockname, + parser_state = ParserState, + proto_state = ProtoState, + enable_stats = EnableStats, + idle_timeout = IdleTimout}}. + +send_fun(WsPid) -> + fun(Packet, Options) -> + Data = emqx_frame:serialize(Packet, Options), + BinSize = iolist_size(Data), + emqx_metrics:trans(inc, 'bytes/sent', BinSize), + emqx_pd:update_counter(send_cnt, 1), + emqx_pd:update_counter(send_oct, BinSize), + WsPid ! {binary, iolist_to_binary(Data)}, + ok + end. + +stat_fun() -> + fun() -> {ok, emqx_pd:get_counter(recv_oct)} end. + +websocket_handle({binary, <<>>}, State) -> + {ok, ensure_stats_timer(State)}; +websocket_handle({binary, [<<>>]}, State) -> + {ok, ensure_stats_timer(State)}; +websocket_handle({binary, Data}, State = #state{parser_state = ParserState, + proto_state = ProtoState}) -> + ?LOG(debug, "RECV ~p", [Data]), + BinSize = iolist_size(Data), + emqx_pd:update_counter(recv_oct, BinSize), + emqx_metrics:trans(inc, 'bytes/received', BinSize), + try emqx_frame:parse(iolist_to_binary(Data), ParserState) of + {more, ParserState1} -> + {ok, State#state{parser_state = ParserState1}}; + {ok, Packet, Rest} -> + emqx_metrics:received(Packet), + emqx_pd:update_counter(recv_cnt, 1), + case emqx_protocol:received(Packet, ProtoState) of + {ok, ProtoState1} -> + websocket_handle({binary, Rest}, reset_parser(State#state{proto_state = ProtoState1})); + {error, Error} -> + ?LOG(error, "Protocol error - ~p", [Error]), + shutdown(Error, State); + {error, Reason, ProtoState1} -> + shutdown(Reason, State#state{proto_state = ProtoState1}); + {stop, Error, ProtoState1} -> + shutdown(Error, State#state{proto_state = ProtoState1}) + end; + {error, Error} -> + ?LOG(error, "Frame error: ~p", [Error]), + shutdown(Error, State) + catch + _:Error -> + ?LOG(error, "Frame error:~p~nFrame data: ~p", [Error, Data]), + shutdown(parse_error, State) + end. + +websocket_info({call, From, info}, State) -> + gen_server:reply(From, info(State)), + {ok, State}; + +websocket_info({call, From, attrs}, State) -> + gen_server:reply(From, attrs(State)), + {ok, State}; + +websocket_info({call, From, stats}, State) -> + gen_server:reply(From, stats(State)), + {ok, State}; + +websocket_info({call, From, kick}, State) -> + gen_server:reply(From, ok), + shutdown(kick, State); + +websocket_info({call, From, session}, State = #state{proto_state = ProtoState}) -> + gen_server:reply(From, emqx_protocol:session(ProtoState)), + {ok, State}; + +websocket_info({deliver, PubOrAck}, State = #state{proto_state = ProtoState}) -> + case emqx_protocol:deliver(PubOrAck, ProtoState) of + {ok, ProtoState1} -> + {ok, ensure_stats_timer(State#state{proto_state = ProtoState1})}; + {error, Reason} -> + shutdown(Reason, State) + end; + +websocket_info({timeout, Timer, emit_stats}, + State = #state{stats_timer = Timer, proto_state = ProtoState}) -> + emqx_metrics:commit(), + emqx_cm:set_conn_stats(emqx_protocol:client_id(ProtoState), stats(State)), + {ok, State#state{stats_timer = undefined}, hibernate}; + +websocket_info({keepalive, start, Interval}, State) -> + ?LOG(debug, "Keepalive at the interval of ~p", [Interval]), + case emqx_keepalive:start(stat_fun(), Interval, {keepalive, check}) of + {ok, KeepAlive} -> + {ok, State#state{keepalive = KeepAlive}}; + {error, Error} -> + ?LOG(warning, "Keepalive error - ~p", [Error]), + shutdown(Error, State) + end; + +websocket_info({keepalive, check}, State = #state{keepalive = KeepAlive}) -> + case emqx_keepalive:check(KeepAlive) of + {ok, KeepAlive1} -> + {ok, State#state{keepalive = KeepAlive1}}; + {error, timeout} -> + ?LOG(debug, "Keepalive Timeout!", []), + shutdown(keepalive_timeout, State); + {error, Error} -> + ?LOG(warning, "Keepalive error - ~p", [Error]), + shutdown(keepalive_error, State) + end; + +websocket_info({shutdown, discard, {ClientId, ByPid}}, State) -> + ?LOG(warning, "discarded by ~s:~p", [ClientId, ByPid]), + shutdown(discard, State); + +websocket_info({shutdown, conflict, {ClientId, NewPid}}, State) -> + ?LOG(warning, "clientid '~s' conflict with ~p", [ClientId, NewPid]), + shutdown(conflict, State); + +websocket_info({binary, Data}, State) -> + {reply, {binary, Data}, State}; + +websocket_info({shutdown, Reason}, State) -> + shutdown(Reason, State); + +websocket_info(Info, State) -> + ?LOG(error, "unexpected info: ~p", [Info]), + {ok, State}. + +terminate(SockError, _Req, #state{keepalive = Keepalive, + proto_state = ProtoState, + shutdown = Shutdown}) -> + + ?LOG(debug, "Terminated for ~p, sockerror: ~p", + [Shutdown, SockError]), + emqx_keepalive:cancel(Keepalive), + case {ProtoState, Shutdown} of + {undefined, _} -> ok; + {_, {shutdown, Reason}} -> + emqx_protocol:shutdown(Reason, ProtoState); + {_, Error} -> + emqx_protocol:shutdown(Error, ProtoState) + end. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +reset_parser(State = #state{proto_state = ProtoState}) -> + State#state{parser_state = emqx_protocol:parser(ProtoState)}. + +ensure_stats_timer(State = #state{enable_stats = true, + stats_timer = undefined, + idle_timeout = IdleTimeout}) -> + State#state{stats_timer = emqx_misc:start_timer(IdleTimeout, emit_stats)}; +ensure_stats_timer(State) -> + State. + +shutdown(Reason, State) -> + {stop, State#state{shutdown = Reason}}. + +wsock_stats() -> + [{Key, emqx_pd:get_counter(Key)} || Key <- ?SOCK_STATS]. + diff --git a/src/emqx_zone.erl b/src/emqx_zone.erl new file mode 100644 index 000000000..d119abe52 --- /dev/null +++ b/src/emqx_zone.erl @@ -0,0 +1,116 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_zone). + +-behaviour(gen_server). + +-include("emqx.hrl"). + +-export([start_link/0]). +-export([get_env/2, get_env/3]). +-export([set_env/3]). +-export([force_reload/0]). +%% for test +-export([stop/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, + code_change/3]). + +-define(TAB, ?MODULE). +-define(SERVER, ?MODULE). + +-spec(start_link() -> emqx_types:startlink_ret()). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +-spec(get_env(emqx_types:zone() | undefined, atom()) -> undefined | term()). +get_env(undefined, Key) -> + emqx_config:get_env(Key); +get_env(Zone, Key) -> + get_env(Zone, Key, undefined). + +-spec(get_env(emqx_types:zone() | undefined, atom(), term()) -> undefined | term()). +get_env(undefined, Key, Def) -> + emqx_config:get_env(Key, Def); +get_env(Zone, Key, Def) -> + try ets:lookup_element(?TAB, {Zone, Key}, 2) + catch error:badarg -> + emqx_config:get_env(Key, Def) + end. + +-spec(set_env(emqx_types:zone(), atom(), term()) -> ok). +set_env(Zone, Key, Val) -> + gen_server:cast(?SERVER, {set_env, Zone, Key, Val}). + +-spec(force_reload() -> ok). +force_reload() -> + gen_server:call(?SERVER, force_reload). + +-spec(stop() -> ok). +stop() -> + gen_server:stop(?SERVER, normal, infinity). + +%%------------------------------------------------------------------------------ +%% gen_server callbacks +%%------------------------------------------------------------------------------ + +init([]) -> + ok = emqx_tables:new(?TAB, [set, {read_concurrency, true}]), + {ok, element(2, handle_info(reload, #{timer => undefined}))}. + +handle_call(force_reload, _From, State) -> + _ = do_reload(), + {reply, ok, State}; + +handle_call(Req, _From, State) -> + emqx_logger:error("[Zone] unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({set_env, Zone, Key, Val}, State) -> + true = ets:insert(?TAB, {{Zone, Key}, Val}), + {noreply, State}; + +handle_cast(Msg, State) -> + emqx_logger:error("[Zone] unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info(reload, State) -> + _ = do_reload(), + {noreply, ensure_reload_timer(State#{timer := undefined}), hibernate}; + +handle_info(Info, State) -> + emqx_logger:error("[Zone] unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +do_reload() -> + [ets:insert(?TAB, [{{Zone, Key}, Val} || {Key, Val} <- Opts]) + || {Zone, Opts} <- emqx_config:get_env(zones, [])]. + +ensure_reload_timer(State = #{timer := undefined}) -> + State#{timer := erlang:send_after(timer:minutes(5), self(), reload)}; +ensure_reload_timer(State) -> + State. + diff --git a/src/gen_server2.erl b/src/gen_server2.erl deleted file mode 100644 index d5127f8d6..000000000 --- a/src/gen_server2.erl +++ /dev/null @@ -1,1361 +0,0 @@ -%% This file is a copy of gen_server.erl from the R13B-1 Erlang/OTP -%% distribution, with the following modifications: -%% -%% 1) the module name is gen_server2 -%% -%% 2) more efficient handling of selective receives in callbacks -%% gen_server2 processes drain their message queue into an internal -%% buffer before invoking any callback module functions. Messages are -%% dequeued from the buffer for processing. Thus the effective message -%% queue of a gen_server2 process is the concatenation of the internal -%% buffer and the real message queue. -%% As a result of the draining, any selective receive invoked inside a -%% callback is less likely to have to scan a large message queue. -%% -%% 3) gen_server2:cast is guaranteed to be order-preserving -%% The original code could reorder messages when communicating with a -%% process on a remote node that was not currently connected. -%% -%% 4) The callback module can optionally implement prioritise_call/4, -%% prioritise_cast/3 and prioritise_info/3. These functions take -%% Message, From, Length and State or just Message, Length and State -%% (where Length is the current number of messages waiting to be -%% processed) and return a single integer representing the priority -%% attached to the message, or 'drop' to ignore it (for -%% prioritise_cast/3 and prioritise_info/3 only). Messages with -%% higher priorities are processed before requests with lower -%% priorities. The default priority is 0. -%% -%% 5) The callback module can optionally implement -%% handle_pre_hibernate/1 and handle_post_hibernate/1. These will be -%% called immediately prior to and post hibernation, respectively. If -%% handle_pre_hibernate returns {hibernate, NewState} then the process -%% will hibernate. If the module does not implement -%% handle_pre_hibernate/1 then the default action is to hibernate. -%% -%% 6) init can return a 4th arg, {backoff, InitialTimeout, -%% MinimumTimeout, DesiredHibernatePeriod} (all in milliseconds, -%% 'infinity' does not make sense here). Then, on all callbacks which -%% can return a timeout (including init), timeout can be -%% 'hibernate'. When this is the case, the current timeout value will -%% be used (initially, the InitialTimeout supplied from init). After -%% this timeout has occurred, hibernation will occur as normal. Upon -%% awaking, a new current timeout value will be calculated. -%% -%% The purpose is that the gen_server2 takes care of adjusting the -%% current timeout value such that the process will increase the -%% timeout value repeatedly if it is unable to sleep for the -%% DesiredHibernatePeriod. If it is able to sleep for the -%% DesiredHibernatePeriod it will decrease the current timeout down to -%% the MinimumTimeout, so that the process is put to sleep sooner (and -%% hopefully stays asleep for longer). In short, should a process -%% using this receive a burst of messages, it should not hibernate -%% between those messages, but as the messages become less frequent, -%% the process will not only hibernate, it will do so sooner after -%% each message. -%% -%% When using this backoff mechanism, normal timeout values (i.e. not -%% 'hibernate') can still be used, and if they are used then the -%% handle_info(timeout, State) will be called as normal. In this case, -%% returning 'hibernate' from handle_info(timeout, State) will not -%% hibernate the process immediately, as it would if backoff wasn't -%% being used. Instead it'll wait for the current timeout as described -%% above. -%% -%% 7) The callback module can return from any of the handle_* -%% functions, a {become, Module, State} triple, or a {become, Module, -%% State, Timeout} quadruple. This allows the gen_server to -%% dynamically change the callback module. The State is the new state -%% which will be passed into any of the callback functions in the new -%% module. Note there is no form also encompassing a reply, thus if -%% you wish to reply in handle_call/3 and change the callback module, -%% you need to use gen_server2:reply/2 to issue the reply -%% manually. The init function can similarly return a 5th argument, -%% Module, in order to dynamically decide the callback module on init. -%% -%% 8) The callback module can optionally implement -%% format_message_queue/2 which is the equivalent of format_status/2 -%% but where the second argument is specifically the priority_queue -%% which contains the prioritised message_queue. -%% -%% 9) The function with_state/2 can be used to debug a process with -%% heavyweight state (without needing to copy the entire state out of -%% process as sys:get_status/1 would). Pass through a function which -%% can be invoked on the state, get back the result. The state is not -%% modified. -%% -%% 10) an mcall/1 function has been added for performing multiple -%% call/3 in parallel. Unlike multi_call, which sends the same request -%% to same-named processes residing on a supplied list of nodes, it -%% operates on name/request pairs, where name is anything accepted by -%% call/3, i.e. a pid, global name, local name, or local name on a -%% particular node. -%% - -%% All modifications are (C) 2009-2013 GoPivotal, Inc. - -%% ``The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved via the world wide web at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% The Initial Developer of the Original Code is Ericsson Utvecklings AB. -%% Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings -%% AB. All Rights Reserved.'' -%% -%% $Id$ -%% --module(gen_server2). - -%%% --------------------------------------------------- -%%% -%%% The idea behind THIS server is that the user module -%%% provides (different) functions to handle different -%%% kind of inputs. -%%% If the Parent process terminates the Module:terminate/2 -%%% function is called. -%%% -%%% The user module should export: -%%% -%%% init(Args) -%%% ==> {ok, State} -%%% {ok, State, Timeout} -%%% {ok, State, Timeout, Backoff} -%%% {ok, State, Timeout, Backoff, Module} -%%% ignore -%%% {stop, Reason} -%%% -%%% handle_call(Msg, {From, Tag}, State) -%%% -%%% ==> {reply, Reply, State} -%%% {reply, Reply, State, Timeout} -%%% {noreply, State} -%%% {noreply, State, Timeout} -%%% {stop, Reason, Reply, State} -%%% Reason = normal | shutdown | Term terminate(State) is called -%%% -%%% handle_cast(Msg, State) -%%% -%%% ==> {noreply, State} -%%% {noreply, State, Timeout} -%%% {stop, Reason, State} -%%% Reason = normal | shutdown | Term terminate(State) is called -%%% -%%% handle_info(Info, State) Info is e.g. {'EXIT', P, R}, {nodedown, N}, ... -%%% -%%% ==> {noreply, State} -%%% {noreply, State, Timeout} -%%% {stop, Reason, State} -%%% Reason = normal | shutdown | Term, terminate(State) is called -%%% -%%% terminate(Reason, State) Let the user module clean up -%%% Reason = normal | shutdown | {shutdown, Term} | Term -%%% always called when server terminates -%%% -%%% ==> ok | Term -%%% -%%% handle_pre_hibernate(State) -%%% -%%% ==> {hibernate, State} -%%% {stop, Reason, State} -%%% Reason = normal | shutdown | Term, terminate(State) is called -%%% -%%% handle_post_hibernate(State) -%%% -%%% ==> {noreply, State} -%%% {stop, Reason, State} -%%% Reason = normal | shutdown | Term, terminate(State) is called -%%% -%%% The work flow (of the server) can be described as follows: -%%% -%%% User module Generic -%%% ----------- ------- -%%% start -----> start -%%% init <----- . -%%% -%%% loop -%%% handle_call <----- . -%%% -----> reply -%%% -%%% handle_cast <----- . -%%% -%%% handle_info <----- . -%%% -%%% terminate <----- . -%%% -%%% -----> reply -%%% -%%% -%%% --------------------------------------------------- - -%% API --export([start/3, start/4, - start_link/3, start_link/4, - call/2, call/3, - cast/2, reply/2, - abcast/2, abcast/3, - multi_call/2, multi_call/3, multi_call/4, - mcall/1, - with_state/2, - enter_loop/3, enter_loop/4, enter_loop/5, enter_loop/6, wake_hib/1]). - -%% System exports --export([system_continue/3, - system_terminate/4, - system_code_change/4, - format_status/2]). - -%% Internal exports --export([init_it/6]). - --import(error_logger, [format/2]). - -%% State record --record(gs2_state, {parent, name, state, mod, time, - timeout_state, queue, debug, prioritisers}). - --ifdef(use_specs). - -%%%========================================================================= -%%% Specs. These exist only to shut up dialyzer's warnings -%%%========================================================================= - --type(gs2_state() :: #gs2_state{}). - --spec(handle_common_termination/3 :: - (any(), atom(), gs2_state()) -> no_return()). --spec(hibernate/1 :: (gs2_state()) -> no_return()). --spec(pre_hibernate/1 :: (gs2_state()) -> no_return()). --spec(system_terminate/4 :: (_, _, _, gs2_state()) -> no_return()). - --type(millis() :: non_neg_integer()). - -%%%========================================================================= -%%% API -%%%========================================================================= - --callback init(Args :: term()) -> - {ok, State :: term()} | - {ok, State :: term(), timeout() | hibernate} | - {ok, State :: term(), timeout() | hibernate, - {backoff, millis(), millis(), millis()}} | - {ok, State :: term(), timeout() | hibernate, - {backoff, millis(), millis(), millis()}, atom()} | - ignore | - {stop, Reason :: term()}. --callback handle_call(Request :: term(), From :: {pid(), Tag :: term()}, - State :: term()) -> - {reply, Reply :: term(), NewState :: term()} | - {reply, Reply :: term(), NewState :: term(), timeout() | hibernate} | - {noreply, NewState :: term()} | - {noreply, NewState :: term(), timeout() | hibernate} | - {stop, Reason :: term(), - Reply :: term(), NewState :: term()}. --callback handle_cast(Request :: term(), State :: term()) -> - {noreply, NewState :: term()} | - {noreply, NewState :: term(), timeout() | hibernate} | - {stop, Reason :: term(), NewState :: term()}. --callback handle_info(Info :: term(), State :: term()) -> - {noreply, NewState :: term()} | - {noreply, NewState :: term(), timeout() | hibernate} | - {stop, Reason :: term(), NewState :: term()}. --callback terminate(Reason :: (normal | shutdown | {shutdown, term()} | term()), - State :: term()) -> - ok | term(). --callback code_change(OldVsn :: (term() | {down, term()}), State :: term(), - Extra :: term()) -> - {ok, NewState :: term()} | {error, Reason :: term()}. - -%% It's not possible to define "optional" -callbacks, so putting specs -%% for handle_pre_hibernate/1 and handle_post_hibernate/1 will result -%% in warnings (the same applied for the behaviour_info before). - --else. - --export([behaviour_info/1]). - -behaviour_info(callbacks) -> - [{init,1},{handle_call,3},{handle_cast,2},{handle_info,2}, - {terminate,2},{code_change,3}]; -behaviour_info(_Other) -> - undefined. - --endif. - -%%% ----------------------------------------------------------------- -%%% Starts a generic server. -%%% start(Mod, Args, Options) -%%% start(Name, Mod, Args, Options) -%%% start_link(Mod, Args, Options) -%%% start_link(Name, Mod, Args, Options) where: -%%% Name ::= {local, atom()} | {global, atom()} -%%% Mod ::= atom(), callback module implementing the 'real' server -%%% Args ::= term(), init arguments (to Mod:init/1) -%%% Options ::= [{timeout, Timeout} | {debug, [Flag]}] -%%% Flag ::= trace | log | {logfile, File} | statistics | debug -%%% (debug == log && statistics) -%%% Returns: {ok, Pid} | -%%% {error, {already_started, Pid}} | -%%% {error, Reason} -%%% ----------------------------------------------------------------- -start(Mod, Args, Options) -> - gen:start(?MODULE, nolink, Mod, Args, Options). - -start(Name, Mod, Args, Options) -> - gen:start(?MODULE, nolink, Name, Mod, Args, Options). - -start_link(Mod, Args, Options) -> - gen:start(?MODULE, link, Mod, Args, Options). - -start_link(Name, Mod, Args, Options) -> - gen:start(?MODULE, link, Name, Mod, Args, Options). - - -%% ----------------------------------------------------------------- -%% Make a call to a generic server. -%% If the server is located at another node, that node will -%% be monitored. -%% If the client is trapping exits and is linked server termination -%% is handled here (? Shall we do that here (or rely on timeouts) ?). -%% ----------------------------------------------------------------- -call(Name, Request) -> - case catch gen:call(Name, '$gen_call', Request) of - {ok,Res} -> - Res; - {'EXIT',Reason} -> - exit({Reason, {?MODULE, call, [Name, Request]}}) - end. - -call(Name, Request, Timeout) -> - case catch gen:call(Name, '$gen_call', Request, Timeout) of - {ok,Res} -> - Res; - {'EXIT',Reason} -> - exit({Reason, {?MODULE, call, [Name, Request, Timeout]}}) - end. - -%% ----------------------------------------------------------------- -%% Make a cast to a generic server. -%% ----------------------------------------------------------------- -cast({global,Name}, Request) -> - catch global:send(Name, cast_msg(Request)), - ok; -cast({Name,Node}=Dest, Request) when is_atom(Name), is_atom(Node) -> - do_cast(Dest, Request); -cast(Dest, Request) when is_atom(Dest) -> - do_cast(Dest, Request); -cast(Dest, Request) when is_pid(Dest) -> - do_cast(Dest, Request). - -do_cast(Dest, Request) -> - do_send(Dest, cast_msg(Request)), - ok. - -cast_msg(Request) -> {'$gen_cast',Request}. - -%% ----------------------------------------------------------------- -%% Send a reply to the client. -%% ----------------------------------------------------------------- -reply({To, Tag}, Reply) -> - catch To ! {Tag, Reply}. - -%% ----------------------------------------------------------------- -%% Asyncronous broadcast, returns nothing, it's just send'n pray -%% ----------------------------------------------------------------- -abcast(Name, Request) when is_atom(Name) -> - do_abcast([node() | nodes()], Name, cast_msg(Request)). - -abcast(Nodes, Name, Request) when is_list(Nodes), is_atom(Name) -> - do_abcast(Nodes, Name, cast_msg(Request)). - -do_abcast([Node|Nodes], Name, Msg) when is_atom(Node) -> - do_send({Name,Node},Msg), - do_abcast(Nodes, Name, Msg); -do_abcast([], _,_) -> abcast. - -%%% ----------------------------------------------------------------- -%%% Make a call to servers at several nodes. -%%% Returns: {[Replies],[BadNodes]} -%%% A Timeout can be given -%%% -%%% A middleman process is used in case late answers arrives after -%%% the timeout. If they would be allowed to glog the callers message -%%% queue, it would probably become confused. Late answers will -%%% now arrive to the terminated middleman and so be discarded. -%%% ----------------------------------------------------------------- -multi_call(Name, Req) - when is_atom(Name) -> - do_multi_call([node() | nodes()], Name, Req, infinity). - -multi_call(Nodes, Name, Req) - when is_list(Nodes), is_atom(Name) -> - do_multi_call(Nodes, Name, Req, infinity). - -multi_call(Nodes, Name, Req, infinity) -> - do_multi_call(Nodes, Name, Req, infinity); -multi_call(Nodes, Name, Req, Timeout) - when is_list(Nodes), is_atom(Name), is_integer(Timeout), Timeout >= 0 -> - do_multi_call(Nodes, Name, Req, Timeout). - -%%% ----------------------------------------------------------------- -%%% Make multiple calls to multiple servers, given pairs of servers -%%% and messages. -%%% Returns: {[{Dest, Reply}], [{Dest, Error}]} -%%% -%%% Dest can be pid() | RegName :: atom() | -%%% {Name :: atom(), Node :: atom()} | {global, Name :: atom()} -%%% -%%% A middleman process is used to avoid clogging up the callers -%%% message queue. -%%% ----------------------------------------------------------------- -mcall(CallSpecs) -> - Tag = make_ref(), - {_, MRef} = spawn_monitor( - fun() -> - Refs = lists:foldl( - fun ({Dest, _Request}=S, Dict) -> - dict:store(do_mcall(S), Dest, Dict) - end, dict:new(), CallSpecs), - collect_replies(Tag, Refs, [], []) - end), - receive - {'DOWN', MRef, _, _, {Tag, Result}} -> Result; - {'DOWN', MRef, _, _, Reason} -> exit(Reason) - end. - -do_mcall({{global,Name}=Dest, Request}) -> - %% whereis_name is simply an ets lookup, and is precisely what - %% global:send/2 does, yet we need a Ref to put in the call to the - %% server, so invoking whereis_name makes a lot more sense here. - case global:whereis_name(Name) of - Pid when is_pid(Pid) -> - MRef = erlang:monitor(process, Pid), - catch msend(Pid, MRef, Request), - MRef; - undefined -> - Ref = make_ref(), - self() ! {'DOWN', Ref, process, Dest, noproc}, - Ref - end; -do_mcall({{Name,Node}=Dest, Request}) when is_atom(Name), is_atom(Node) -> - {_Node, MRef} = start_monitor(Node, Name), %% NB: we don't handle R6 - catch msend(Dest, MRef, Request), - MRef; -do_mcall({Dest, Request}) when is_atom(Dest); is_pid(Dest) -> - MRef = erlang:monitor(process, Dest), - catch msend(Dest, MRef, Request), - MRef. - -msend(Dest, MRef, Request) -> - erlang:send(Dest, {'$gen_call', {self(), MRef}, Request}, [noconnect]). - -collect_replies(Tag, Refs, Replies, Errors) -> - case dict:size(Refs) of - 0 -> exit({Tag, {Replies, Errors}}); - _ -> receive - {MRef, Reply} -> - {Refs1, Replies1} = handle_call_result(MRef, Reply, - Refs, Replies), - collect_replies(Tag, Refs1, Replies1, Errors); - {'DOWN', MRef, _, _, Reason} -> - Reason1 = case Reason of - noconnection -> nodedown; - _ -> Reason - end, - {Refs1, Errors1} = handle_call_result(MRef, Reason1, - Refs, Errors), - collect_replies(Tag, Refs1, Replies, Errors1) - end - end. - -handle_call_result(MRef, Result, Refs, AccList) -> - %% we avoid the mailbox scanning cost of a call to erlang:demonitor/{1,2} - %% here, so we must cope with MRefs that we've already seen and erased - case dict:find(MRef, Refs) of - {ok, Pid} -> {dict:erase(MRef, Refs), [{Pid, Result}|AccList]}; - _ -> {Refs, AccList} - end. - -%% ----------------------------------------------------------------- -%% Apply a function to a generic server's state. -%% ----------------------------------------------------------------- -with_state(Name, Fun) -> - case catch gen:call(Name, '$with_state', Fun, infinity) of - {ok,Res} -> - Res; - {'EXIT',Reason} -> - exit({Reason, {?MODULE, with_state, [Name, Fun]}}) - end. - -%%----------------------------------------------------------------- -%% enter_loop(Mod, Options, State, , , ) ->_ -%% -%% Description: Makes an existing process into a gen_server. -%% The calling process will enter the gen_server receive -%% loop and become a gen_server process. -%% The process *must* have been started using one of the -%% start functions in proc_lib, see proc_lib(3). -%% The user is responsible for any initialization of the -%% process, including registering a name for it. -%%----------------------------------------------------------------- -enter_loop(Mod, Options, State) -> - enter_loop(Mod, Options, State, self(), infinity, undefined). - -enter_loop(Mod, Options, State, Backoff = {backoff, _, _ , _}) -> - enter_loop(Mod, Options, State, self(), infinity, Backoff); - -enter_loop(Mod, Options, State, ServerName = {_, _}) -> - enter_loop(Mod, Options, State, ServerName, infinity, undefined); - -enter_loop(Mod, Options, State, Timeout) -> - enter_loop(Mod, Options, State, self(), Timeout, undefined). - -enter_loop(Mod, Options, State, ServerName, Backoff = {backoff, _, _, _}) -> - enter_loop(Mod, Options, State, ServerName, infinity, Backoff); - -enter_loop(Mod, Options, State, ServerName, Timeout) -> - enter_loop(Mod, Options, State, ServerName, Timeout, undefined). - -enter_loop(Mod, Options, State, ServerName, Timeout, Backoff) -> - Name = get_proc_name(ServerName), - Parent = get_parent(), - Debug = debug_options(Name, Options), - Queue = priority_queue:new(), - Backoff1 = extend_backoff(Backoff), - loop(find_prioritisers( - #gs2_state { parent = Parent, name = Name, state = State, - mod = Mod, time = Timeout, timeout_state = Backoff1, - queue = Queue, debug = Debug })). - -%%%======================================================================== -%%% Gen-callback functions -%%%======================================================================== - -%%% --------------------------------------------------- -%%% Initiate the new process. -%%% Register the name using the Rfunc function -%%% Calls the Mod:init/Args function. -%%% Finally an acknowledge is sent to Parent and the main -%%% loop is entered. -%%% --------------------------------------------------- -init_it(Starter, self, Name, Mod, Args, Options) -> - init_it(Starter, self(), Name, Mod, Args, Options); -init_it(Starter, Parent, Name0, Mod, Args, Options) -> - Name = name(Name0), - Debug = debug_options(Name, Options), - Queue = priority_queue:new(), - GS2State = find_prioritisers( - #gs2_state { parent = Parent, - name = Name, - mod = Mod, - queue = Queue, - debug = Debug }), - case catch Mod:init(Args) of - {ok, State} -> - proc_lib:init_ack(Starter, {ok, self()}), - loop(GS2State #gs2_state { state = State, - time = infinity, - timeout_state = undefined }); - {ok, State, Timeout} -> - proc_lib:init_ack(Starter, {ok, self()}), - loop(GS2State #gs2_state { state = State, - time = Timeout, - timeout_state = undefined }); - {ok, State, Timeout, Backoff = {backoff, _, _, _}} -> - Backoff1 = extend_backoff(Backoff), - proc_lib:init_ack(Starter, {ok, self()}), - loop(GS2State #gs2_state { state = State, - time = Timeout, - timeout_state = Backoff1 }); - {ok, State, Timeout, Backoff = {backoff, _, _, _}, Mod1} -> - Backoff1 = extend_backoff(Backoff), - proc_lib:init_ack(Starter, {ok, self()}), - loop(find_prioritisers( - GS2State #gs2_state { mod = Mod1, - state = State, - time = Timeout, - timeout_state = Backoff1 })); - {stop, Reason} -> - %% For consistency, we must make sure that the - %% registered name (if any) is unregistered before - %% the parent process is notified about the failure. - %% (Otherwise, the parent process could get - %% an 'already_started' error if it immediately - %% tried starting the process again.) - unregister_name(Name0), - proc_lib:init_ack(Starter, {error, Reason}), - exit(Reason); - ignore -> - unregister_name(Name0), - proc_lib:init_ack(Starter, ignore), - exit(normal); - {'EXIT', Reason} -> - unregister_name(Name0), - proc_lib:init_ack(Starter, {error, Reason}), - exit(Reason); - Else -> - Error = {bad_return_value, Else}, - proc_lib:init_ack(Starter, {error, Error}), - exit(Error) - end. - -name({local,Name}) -> Name; -name({global,Name}) -> Name; -%% name(Pid) when is_pid(Pid) -> Pid; -%% when R12 goes away, drop the line beneath and uncomment the line above -name(Name) -> Name. - -unregister_name({local,Name}) -> - _ = (catch unregister(Name)); -unregister_name({global,Name}) -> - _ = global:unregister_name(Name); -unregister_name(Pid) when is_pid(Pid) -> - Pid; -%% Under R12 let's just ignore it, as we have a single term as Name. -%% On R13 it will never get here, as we get tuple with 'local/global' atom. -unregister_name(_Name) -> ok. - -extend_backoff(undefined) -> - undefined; -extend_backoff({backoff, InitialTimeout, MinimumTimeout, DesiredHibPeriod}) -> - {backoff, InitialTimeout, MinimumTimeout, DesiredHibPeriod, rand:seed(exsplus)}. - -%%%======================================================================== -%%% Internal functions -%%%======================================================================== -%%% --------------------------------------------------- -%%% The MAIN loop. -%%% --------------------------------------------------- -loop(GS2State = #gs2_state { time = hibernate, - timeout_state = undefined, - queue = Queue }) -> - case priority_queue:is_empty(Queue) of - true -> - pre_hibernate(GS2State); - false -> - process_next_msg(GS2State) - end; - -loop(GS2State) -> - process_next_msg(drain(GS2State)). - -drain(GS2State) -> - receive - Input -> drain(in(Input, GS2State)) - after 0 -> GS2State - end. - -process_next_msg(GS2State = #gs2_state { time = Time, - timeout_state = TimeoutState, - queue = Queue }) -> - case priority_queue:out(Queue) of - {{value, Msg}, Queue1} -> - process_msg(Msg, GS2State #gs2_state { queue = Queue1 }); - {empty, Queue1} -> - {Time1, HibOnTimeout} - = case {Time, TimeoutState} of - {hibernate, {backoff, Current, _Min, _Desired, _RSt}} -> - {Current, true}; - {hibernate, _} -> - %% wake_hib/7 will set Time to hibernate. If - %% we were woken and didn't receive a msg - %% then we will get here and need a sensible - %% value for Time1, otherwise we crash. - %% R13B1 always waits infinitely when waking - %% from hibernation, so that's what we do - %% here too. - {infinity, false}; - _ -> {Time, false} - end, - receive - Input -> - %% Time could be 'hibernate' here, so *don't* call loop - process_next_msg( - drain(in(Input, GS2State #gs2_state { queue = Queue1 }))) - after Time1 -> - case HibOnTimeout of - true -> - pre_hibernate( - GS2State #gs2_state { queue = Queue1 }); - false -> - process_msg(timeout, - GS2State #gs2_state { queue = Queue1 }) - end - end - end. - -wake_hib(GS2State = #gs2_state { timeout_state = TS }) -> - TimeoutState1 = case TS of - undefined -> - undefined; - {SleptAt, TimeoutState} -> - adjust_timeout_state(SleptAt, - erlang:monotonic_time(), - TimeoutState) - end, - post_hibernate( - drain(GS2State #gs2_state { timeout_state = TimeoutState1 })). - -hibernate(GS2State = #gs2_state { timeout_state = TimeoutState }) -> - TS = case TimeoutState of - undefined -> undefined; - {backoff, _, _, _, _} -> {erlang:monotonic_time(), - TimeoutState} - end, - proc_lib:hibernate(?MODULE, wake_hib, - [GS2State #gs2_state { timeout_state = TS }]). - -pre_hibernate(GS2State = #gs2_state { state = State, - mod = Mod }) -> - case erlang:function_exported(Mod, handle_pre_hibernate, 1) of - true -> - case catch Mod:handle_pre_hibernate(State) of - {hibernate, NState} -> - hibernate(GS2State #gs2_state { state = NState } ); - Reply -> - handle_common_termination(Reply, pre_hibernate, GS2State) - end; - false -> - hibernate(GS2State) - end. - -post_hibernate(GS2State = #gs2_state { state = State, - mod = Mod }) -> - case erlang:function_exported(Mod, handle_post_hibernate, 1) of - true -> - case catch Mod:handle_post_hibernate(State) of - {noreply, NState} -> - process_next_msg(GS2State #gs2_state { state = NState, - time = infinity }); - {noreply, NState, Time} -> - process_next_msg(GS2State #gs2_state { state = NState, - time = Time }); - Reply -> - handle_common_termination(Reply, post_hibernate, GS2State) - end; - false -> - %% use hibernate here, not infinity. This matches - %% R13B. The key is that we should be able to get through - %% to process_msg calling sys:handle_system_msg with Time - %% still set to hibernate, iff that msg is the very msg - %% that woke us up (or the first msg we receive after - %% waking up). - process_next_msg(GS2State #gs2_state { time = hibernate }) - end. - -adjust_timeout_state(SleptAt, AwokeAt, {backoff, CurrentTO, MinimumTO, - DesiredHibPeriod, RandomState}) -> - NapLengthMicros = erlang:convert_time_unit(AwokeAt - SleptAt, - native, micro_seconds), - CurrentMicros = CurrentTO * 1000, - MinimumMicros = MinimumTO * 1000, - DesiredHibMicros = DesiredHibPeriod * 1000, - GapBetweenMessagesMicros = NapLengthMicros + CurrentMicros, - Base = - %% If enough time has passed between the last two messages then we - %% should consider sleeping sooner. Otherwise stay awake longer. - case GapBetweenMessagesMicros > (MinimumMicros + DesiredHibMicros) of - true -> lists:max([MinimumTO, CurrentTO div 2]); - false -> CurrentTO - end, - {Extra, RandomState1} = rand:uniform_s(Base, RandomState), - CurrentTO1 = Base + Extra, - {backoff, CurrentTO1, MinimumTO, DesiredHibPeriod, RandomState1}. - -in({'$gen_cast', Msg} = Input, - GS2State = #gs2_state { prioritisers = {_, F, _} }) -> - in(Input, F(Msg, GS2State), GS2State); -in({'$gen_call', From, Msg} = Input, - GS2State = #gs2_state { prioritisers = {F, _, _} }) -> - in(Input, F(Msg, From, GS2State), GS2State); -in({'$with_state', _From, _Fun} = Input, GS2State) -> - in(Input, 0, GS2State); -in({'EXIT', Parent, _R} = Input, GS2State = #gs2_state { parent = Parent }) -> - in(Input, infinity, GS2State); -in({system, _From, _Req} = Input, GS2State) -> - in(Input, infinity, GS2State); -in(Input, GS2State = #gs2_state { prioritisers = {_, _, F} }) -> - in(Input, F(Input, GS2State), GS2State). - -in(_Input, drop, GS2State) -> - GS2State; - -in(Input, Priority, GS2State = #gs2_state { queue = Queue }) -> - GS2State # gs2_state { queue = priority_queue:in(Input, Priority, Queue) }. - -process_msg({system, From, Req}, - GS2State = #gs2_state { parent = Parent, debug = Debug }) -> - %% gen_server puts Hib on the end as the 7th arg, but that version - %% of the fun seems not to be documented so leaving out for now. - sys:handle_system_msg(Req, From, Parent, ?MODULE, Debug, GS2State); -process_msg({'$with_state', From, Fun}, - GS2State = #gs2_state{state = State}) -> - reply(From, catch Fun(State)), - loop(GS2State); -process_msg({'EXIT', Parent, Reason} = Msg, - GS2State = #gs2_state { parent = Parent }) -> - terminate(Reason, Msg, GS2State); -process_msg(Msg, GS2State = #gs2_state { debug = [] }) -> - handle_msg(Msg, GS2State); -process_msg(Msg, GS2State = #gs2_state { name = Name, debug = Debug }) -> - Debug1 = sys:handle_debug(Debug, fun print_event/3, Name, {in, Msg}), - handle_msg(Msg, GS2State #gs2_state { debug = Debug1 }). - -%%% --------------------------------------------------- -%%% Send/recive functions -%%% --------------------------------------------------- -do_send(Dest, Msg) -> - catch erlang:send(Dest, Msg). - -do_multi_call(Nodes, Name, Req, infinity) -> - Tag = make_ref(), - Monitors = send_nodes(Nodes, Name, Tag, Req), - rec_nodes(Tag, Monitors, Name, undefined); -do_multi_call(Nodes, Name, Req, Timeout) -> - Tag = make_ref(), - Caller = self(), - Receiver = - spawn( - fun () -> - %% Middleman process. Should be unsensitive to regular - %% exit signals. The sychronization is needed in case - %% the receiver would exit before the caller started - %% the monitor. - process_flag(trap_exit, true), - Mref = erlang:monitor(process, Caller), - receive - {Caller,Tag} -> - Monitors = send_nodes(Nodes, Name, Tag, Req), - TimerId = erlang:start_timer(Timeout, self(), ok), - Result = rec_nodes(Tag, Monitors, Name, TimerId), - exit({self(),Tag,Result}); - {'DOWN',Mref,_,_,_} -> - %% Caller died before sending us the go-ahead. - %% Give up silently. - exit(normal) - end - end), - Mref = erlang:monitor(process, Receiver), - Receiver ! {self(),Tag}, - receive - {'DOWN',Mref,_,_,{Receiver,Tag,Result}} -> - Result; - {'DOWN',Mref,_,_,Reason} -> - %% The middleman code failed. Or someone did - %% exit(_, kill) on the middleman process => Reason==killed - exit(Reason) - end. - -send_nodes(Nodes, Name, Tag, Req) -> - send_nodes(Nodes, Name, Tag, Req, []). - -send_nodes([Node|Tail], Name, Tag, Req, Monitors) - when is_atom(Node) -> - Monitor = start_monitor(Node, Name), - %% Handle non-existing names in rec_nodes. - catch {Name, Node} ! {'$gen_call', {self(), {Tag, Node}}, Req}, - send_nodes(Tail, Name, Tag, Req, [Monitor | Monitors]); -send_nodes([_Node|Tail], Name, Tag, Req, Monitors) -> - %% Skip non-atom Node - send_nodes(Tail, Name, Tag, Req, Monitors); -send_nodes([], _Name, _Tag, _Req, Monitors) -> - Monitors. - -%% Against old nodes: -%% If no reply has been delivered within 2 secs. (per node) check that -%% the server really exists and wait for ever for the answer. -%% -%% Against contemporary nodes: -%% Wait for reply, server 'DOWN', or timeout from TimerId. - -rec_nodes(Tag, Nodes, Name, TimerId) -> - rec_nodes(Tag, Nodes, Name, [], [], 2000, TimerId). - -rec_nodes(Tag, [{N,R}|Tail], Name, Badnodes, Replies, Time, TimerId ) -> - receive - {'DOWN', R, _, _, _} -> - rec_nodes(Tag, Tail, Name, [N|Badnodes], Replies, Time, TimerId); - {{Tag, N}, Reply} -> %% Tag is bound !!! - unmonitor(R), - rec_nodes(Tag, Tail, Name, Badnodes, - [{N,Reply}|Replies], Time, TimerId); - {timeout, TimerId, _} -> - unmonitor(R), - %% Collect all replies that already have arrived - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies) - end; -rec_nodes(Tag, [N|Tail], Name, Badnodes, Replies, Time, TimerId) -> - %% R6 node - receive - {nodedown, N} -> - monitor_node(N, false), - rec_nodes(Tag, Tail, Name, [N|Badnodes], Replies, 2000, TimerId); - {{Tag, N}, Reply} -> %% Tag is bound !!! - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - rec_nodes(Tag, Tail, Name, Badnodes, - [{N,Reply}|Replies], 2000, TimerId); - {timeout, TimerId, _} -> - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - %% Collect all replies that already have arrived - rec_nodes_rest(Tag, Tail, Name, [N | Badnodes], Replies) - after Time -> - case rpc:call(N, erlang, whereis, [Name]) of - Pid when is_pid(Pid) -> % It exists try again. - rec_nodes(Tag, [N|Tail], Name, Badnodes, - Replies, infinity, TimerId); - _ -> % badnode - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - rec_nodes(Tag, Tail, Name, [N|Badnodes], - Replies, 2000, TimerId) - end - end; -rec_nodes(_, [], _, Badnodes, Replies, _, TimerId) -> - case catch erlang:cancel_timer(TimerId) of - false -> % It has already sent it's message - receive - {timeout, TimerId, _} -> ok - after 0 -> - ok - end; - _ -> % Timer was cancelled, or TimerId was 'undefined' - ok - end, - {Replies, Badnodes}. - -%% Collect all replies that already have arrived -rec_nodes_rest(Tag, [{N,R}|Tail], Name, Badnodes, Replies) -> - receive - {'DOWN', R, _, _, _} -> - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies); - {{Tag, N}, Reply} -> %% Tag is bound !!! - unmonitor(R), - rec_nodes_rest(Tag, Tail, Name, Badnodes, [{N,Reply}|Replies]) - after 0 -> - unmonitor(R), - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies) - end; -rec_nodes_rest(Tag, [N|Tail], Name, Badnodes, Replies) -> - %% R6 node - receive - {nodedown, N} -> - monitor_node(N, false), - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies); - {{Tag, N}, Reply} -> %% Tag is bound !!! - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - rec_nodes_rest(Tag, Tail, Name, Badnodes, [{N,Reply}|Replies]) - after 0 -> - receive {nodedown, N} -> ok after 0 -> ok end, - monitor_node(N, false), - rec_nodes_rest(Tag, Tail, Name, [N|Badnodes], Replies) - end; -rec_nodes_rest(_Tag, [], _Name, Badnodes, Replies) -> - {Replies, Badnodes}. - - -%%% --------------------------------------------------- -%%% Monitor functions -%%% --------------------------------------------------- - -start_monitor(Node, Name) when is_atom(Node), is_atom(Name) -> - if node() =:= nonode@nohost, Node =/= nonode@nohost -> - Ref = make_ref(), - self() ! {'DOWN', Ref, process, {Name, Node}, noconnection}, - {Node, Ref}; - true -> - case catch erlang:monitor(process, {Name, Node}) of - {'EXIT', _} -> - %% Remote node is R6 - monitor_node(Node, true), - Node; - Ref when is_reference(Ref) -> - {Node, Ref} - end - end. - -%% Cancels a monitor started with Ref=erlang:monitor(_, _). -unmonitor(Ref) when is_reference(Ref) -> - erlang:demonitor(Ref), - receive - {'DOWN', Ref, _, _, _} -> - true - after 0 -> - true - end. - -%%% --------------------------------------------------- -%%% Message handling functions -%%% --------------------------------------------------- - -dispatch({'$gen_cast', Msg}, Mod, State) -> - Mod:handle_cast(Msg, State); -dispatch(Info, Mod, State) -> - Mod:handle_info(Info, State). - -common_reply(_Name, From, Reply, _NState, [] = _Debug) -> - reply(From, Reply), - []; -common_reply(Name, {To, _Tag} = From, Reply, NState, Debug) -> - reply(From, Reply), - sys:handle_debug(Debug, fun print_event/3, Name, {out, Reply, To, NState}). - -common_noreply(_Name, _NState, [] = _Debug) -> - []; -common_noreply(Name, NState, Debug) -> - sys:handle_debug(Debug, fun print_event/3, Name, {noreply, NState}). - -common_become(_Name, _Mod, _NState, [] = _Debug) -> - []; -common_become(Name, Mod, NState, Debug) -> - sys:handle_debug(Debug, fun print_event/3, Name, {become, Mod, NState}). - -handle_msg({'$gen_call', From, Msg}, GS2State = #gs2_state { mod = Mod, - state = State, - name = Name, - debug = Debug }) -> - case catch Mod:handle_call(Msg, From, State) of - {reply, Reply, NState} -> - Debug1 = common_reply(Name, From, Reply, NState, Debug), - loop(GS2State #gs2_state { state = NState, - time = infinity, - debug = Debug1 }); - {reply, Reply, NState, Time1} -> - Debug1 = common_reply(Name, From, Reply, NState, Debug), - loop(GS2State #gs2_state { state = NState, - time = Time1, - debug = Debug1}); - {stop, Reason, Reply, NState} -> - {'EXIT', R} = - (catch terminate(Reason, Msg, - GS2State #gs2_state { state = NState })), - common_reply(Name, From, Reply, NState, Debug), - exit(R); - Other -> - handle_common_reply(Other, Msg, GS2State) - end; -handle_msg(Msg, GS2State = #gs2_state { mod = Mod, state = State }) -> - Reply = (catch dispatch(Msg, Mod, State)), - handle_common_reply(Reply, Msg, GS2State). - -handle_common_reply(Reply, Msg, GS2State = #gs2_state { name = Name, - debug = Debug}) -> - case Reply of - {noreply, NState} -> - Debug1 = common_noreply(Name, NState, Debug), - loop(GS2State #gs2_state {state = NState, - time = infinity, - debug = Debug1}); - {noreply, NState, Time1} -> - Debug1 = common_noreply(Name, NState, Debug), - loop(GS2State #gs2_state {state = NState, - time = Time1, - debug = Debug1}); - {become, Mod, NState} -> - Debug1 = common_become(Name, Mod, NState, Debug), - loop(find_prioritisers( - GS2State #gs2_state { mod = Mod, - state = NState, - time = infinity, - debug = Debug1 })); - {become, Mod, NState, Time1} -> - Debug1 = common_become(Name, Mod, NState, Debug), - loop(find_prioritisers( - GS2State #gs2_state { mod = Mod, - state = NState, - time = Time1, - debug = Debug1 })); - _ -> - handle_common_termination(Reply, Msg, GS2State) - end. - -handle_common_termination(Reply, Msg, GS2State) -> - case Reply of - {stop, Reason, NState} -> - terminate(Reason, Msg, GS2State #gs2_state { state = NState }); - {'EXIT', What} -> - terminate(What, Msg, GS2State); - _ -> - terminate({bad_return_value, Reply}, Msg, GS2State) - end. - -%%----------------------------------------------------------------- -%% Callback functions for system messages handling. -%%----------------------------------------------------------------- -system_continue(Parent, Debug, GS2State) -> - loop(GS2State #gs2_state { parent = Parent, debug = Debug }). - -system_terminate(Reason, _Parent, Debug, GS2State) -> - terminate(Reason, [], GS2State #gs2_state { debug = Debug }). - -system_code_change(GS2State = #gs2_state { mod = Mod, - state = State }, - _Module, OldVsn, Extra) -> - case catch Mod:code_change(OldVsn, State, Extra) of - {ok, NewState} -> - NewGS2State = find_prioritisers( - GS2State #gs2_state { state = NewState }), - {ok, [NewGS2State]}; - Else -> - Else - end. - -%%----------------------------------------------------------------- -%% Format debug messages. Print them as the call-back module sees -%% them, not as the real erlang messages. Use trace for that. -%%----------------------------------------------------------------- -print_event(Dev, {in, Msg}, Name) -> - case Msg of - {'$gen_call', {From, _Tag}, Call} -> - io:format(Dev, "*DBG* ~p got call ~p from ~w~n", - [Name, Call, From]); - {'$gen_cast', Cast} -> - io:format(Dev, "*DBG* ~p got cast ~p~n", - [Name, Cast]); - _ -> - io:format(Dev, "*DBG* ~p got ~p~n", [Name, Msg]) - end; -print_event(Dev, {out, Msg, To, State}, Name) -> - io:format(Dev, "*DBG* ~p sent ~p to ~w, new state ~w~n", - [Name, Msg, To, State]); -print_event(Dev, {noreply, State}, Name) -> - io:format(Dev, "*DBG* ~p new state ~w~n", [Name, State]); -print_event(Dev, Event, Name) -> - io:format(Dev, "*DBG* ~p dbg ~p~n", [Name, Event]). - - -%%% --------------------------------------------------- -%%% Terminate the server. -%%% --------------------------------------------------- - -terminate(Reason, Msg, #gs2_state { name = Name, - mod = Mod, - state = State, - debug = Debug }) -> - case catch Mod:terminate(Reason, State) of - {'EXIT', R} -> - error_info(R, Reason, Name, Msg, State, Debug), - exit(R); - _ -> - case Reason of - normal -> - exit(normal); - shutdown -> - exit(shutdown); - {shutdown,_}=Shutdown -> - exit(Shutdown); - _ -> - error_info(Reason, undefined, Name, Msg, State, Debug), - exit(Reason) - end - end. - -error_info(_Reason, _RootCause, application_controller, _Msg, _State, _Debug) -> - %% OTP-5811 Don't send an error report if it's the system process - %% application_controller which is terminating - let init take care - %% of it instead - ok; -error_info(Reason, RootCause, Name, Msg, State, Debug) -> - Reason1 = error_reason(Reason), - Fmt = - "** Generic server ~p terminating~n" - "** Last message in was ~p~n" - "** When Server state == ~p~n" - "** Reason for termination == ~n** ~p~n", - case RootCause of - undefined -> format(Fmt, [Name, Msg, State, Reason1]); - _ -> format(Fmt ++ "** In 'terminate' callback " - "with reason ==~n** ~p~n", - [Name, Msg, State, Reason1, - error_reason(RootCause)]) - end, - sys:print_log(Debug), - ok. - -error_reason({undef,[{M,F,A}|MFAs]} = Reason) -> - case code:is_loaded(M) of - false -> {'module could not be loaded',[{M,F,A}|MFAs]}; - _ -> case erlang:function_exported(M, F, length(A)) of - true -> Reason; - false -> {'function not exported',[{M,F,A}|MFAs]} - end - end; -error_reason(Reason) -> - Reason. - -%%% --------------------------------------------------- -%%% Misc. functions. -%%% --------------------------------------------------- - -opt(Op, [{Op, Value}|_]) -> - {ok, Value}; -opt(Op, [_|Options]) -> - opt(Op, Options); -opt(_, []) -> - false. - -debug_options(Name, Opts) -> - case opt(debug, Opts) of - {ok, Options} -> dbg_options(Name, Options); - _ -> dbg_options(Name, []) - end. - -dbg_options(Name, []) -> - Opts = - case init:get_argument(generic_debug) of - error -> - []; - _ -> - [log, statistics] - end, - dbg_opts(Name, Opts); -dbg_options(Name, Opts) -> - dbg_opts(Name, Opts). - -dbg_opts(Name, Opts) -> - case catch sys:debug_options(Opts) of - {'EXIT',_} -> - format("~p: ignoring erroneous debug options - ~p~n", - [Name, Opts]), - []; - Dbg -> - Dbg - end. - -get_proc_name(Pid) when is_pid(Pid) -> - Pid; -get_proc_name({local, Name}) -> - case process_info(self(), registered_name) of - {registered_name, Name} -> - Name; - {registered_name, _Name} -> - exit(process_not_registered); - [] -> - exit(process_not_registered) - end; -get_proc_name({global, Name}) -> - case whereis_name(Name) of - undefined -> - exit(process_not_registered_globally); - Pid when Pid =:= self() -> - Name; - _Pid -> - exit(process_not_registered_globally) - end. - -get_parent() -> - case get('$ancestors') of - [Parent | _] when is_pid(Parent)-> - Parent; - [Parent | _] when is_atom(Parent)-> - name_to_pid(Parent); - _ -> - exit(process_was_not_started_by_proc_lib) - end. - -name_to_pid(Name) -> - case whereis(Name) of - undefined -> - case whereis_name(Name) of - undefined -> - exit(could_not_find_registerd_name); - Pid -> - Pid - end; - Pid -> - Pid - end. - -whereis_name(Name) -> - case ets:lookup(global_names, Name) of - [{_Name, Pid, _Method, _RPid, _Ref}] -> - if node(Pid) == node() -> - case is_process_alive(Pid) of - true -> Pid; - false -> undefined - end; - true -> - Pid - end; - [] -> undefined - end. - -find_prioritisers(GS2State = #gs2_state { mod = Mod }) -> - PCall = function_exported_or_default(Mod, 'prioritise_call', 4, - fun (_Msg, _From, _State) -> 0 end), - PCast = function_exported_or_default(Mod, 'prioritise_cast', 3, - fun (_Msg, _State) -> 0 end), - PInfo = function_exported_or_default(Mod, 'prioritise_info', 3, - fun (_Msg, _State) -> 0 end), - GS2State #gs2_state { prioritisers = {PCall, PCast, PInfo} }. - -function_exported_or_default(Mod, Fun, Arity, Default) -> - case erlang:function_exported(Mod, Fun, Arity) of - true -> case Arity of - 3 -> fun (Msg, GS2State = #gs2_state { queue = Queue, - state = State }) -> - Length = priority_queue:len(Queue), - case catch Mod:Fun(Msg, Length, State) of - drop -> - drop; - Res when is_integer(Res) -> - Res; - Err -> - handle_common_termination(Err, Msg, GS2State) - end - end; - 4 -> fun (Msg, From, GS2State = #gs2_state { queue = Queue, - state = State }) -> - Length = priority_queue:len(Queue), - case catch Mod:Fun(Msg, From, Length, State) of - Res when is_integer(Res) -> - Res; - Err -> - handle_common_termination(Err, Msg, GS2State) - end - end - end; - false -> Default - end. - -%%----------------------------------------------------------------- -%% Status information -%%----------------------------------------------------------------- -format_status(Opt, StatusData) -> - [PDict, SysState, Parent, Debug, - #gs2_state{name = Name, state = State, mod = Mod, queue = Queue}] = - StatusData, - NameTag = if is_pid(Name) -> - pid_to_list(Name); - is_atom(Name) -> - Name - end, - Header = lists:concat(["Status for generic server ", NameTag]), - Log = sys:get_debug(log, Debug, []), - Specfic = callback(Mod, format_status, [Opt, [PDict, State]], - fun () -> [{data, [{"State", State}]}] end), - Messages = callback(Mod, format_message_queue, [Opt, Queue], - fun () -> priority_queue:to_list(Queue) end), - [{header, Header}, - {data, [{"Status", SysState}, - {"Parent", Parent}, - {"Logged events", Log}, - {"Queued messages", Messages}]} | - Specfic]. - -callback(Mod, FunName, Args, DefaultThunk) -> - case erlang:function_exported(Mod, FunName, length(Args)) of - true -> case catch apply(Mod, FunName, Args) of - {'EXIT', _} -> DefaultThunk(); - Success -> Success - end; - false -> DefaultThunk() - end. diff --git a/src/lager_emqtt_backend.erl b/src/lager_emqtt_backend.erl deleted file mode 100644 index 1ceb9785e..000000000 --- a/src/lager_emqtt_backend.erl +++ /dev/null @@ -1,88 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(lager_emqtt_backend). - --author("Feng Lee "). - --behaviour(gen_event). - --include_lib("lager/include/lager.hrl"). - --export([init/1, handle_call/2, handle_event/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {level :: {'mask', integer()}, - formatter :: atom(), - format_config :: any()}). - --define(DEFAULT_FORMAT, [time, " ", pid, " [",severity, "] ", message]). - -init([Level]) when is_atom(Level) -> - init(Level); - -init(Level) when is_atom(Level) -> - init([Level,{lager_default_formatter, ?DEFAULT_FORMAT}]); - -init([Level,{Formatter, FormatterConfig}]) when is_atom(Formatter) -> - Levels = lager_util:config_to_mask(Level), - {ok, #state{level = Levels, formatter = Formatter, - format_config = FormatterConfig}}. - -handle_call(get_loglevel, #state{level = Level} = State) -> - {ok, Level, State}; - -handle_call({set_loglevel, Level}, State) -> - try lager_util:config_to_mask(Level) of - Levels -> {ok, ok, State#state{level = Levels}} - catch - _:_ -> {ok, {error, bad_log_level}, State} - end; - -handle_call(_Request, State) -> - {ok, ok, State}. - -handle_event({log, Message}, State = #state{level = L}) -> - case lager_util:is_loggable(Message, L, ?MODULE) of - true -> - publish_log(Message, State); - false -> - {ok, State} - end; - -handle_event(_Event, State) -> - {ok, State}. - -handle_info(_Info, State) -> - {ok, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -publish_log(Message, State = #state{formatter = Formatter, - format_config = FormatConfig}) -> - Severity = lager_msg:severity(Message), - Payload = Formatter:format(Message, FormatConfig), - Msg = emqttd_message:make(log, topic(Severity), iolist_to_binary(Payload)), - emqttd:publish(emqttd_message:set_flag(sys, Msg)), - {ok, State}. - -topic(Severity) -> - emqttd_topic:systop(list_to_binary(lists:concat(['logs/', Severity]))). - diff --git a/test/emqttd_SUITE.erl b/test/emqttd_SUITE.erl deleted file mode 100644 index 360905859..000000000 --- a/test/emqttd_SUITE.erl +++ /dev/null @@ -1,667 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_SUITE). - --compile(export_all). - --include("emqttd.hrl"). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("common_test/include/ct.hrl"). - --define(APP, emqttd). - --define(CONTENT_TYPE, "application/json"). - --define(MQTT_SSL_TWOWAY, [{cacertfile, "certs/cacert.pem"}, - {verify, verify_peer}, - {fail_if_no_peer_cert, true}]). - --define(MQTT_SSL_CLIENT, [{keyfile, "certs/client-key.pem"}, - {cacertfile, "certs/cacert.pem"}, - {certfile, "certs/client-cert.pem"}]). - --define(URL, "http://localhost:8080/api/v2/"). - --define(APPL_JSON, "application/json"). - --define(PRINT(PATH), lists:flatten(io_lib:format(PATH, [atom_to_list(node())]))). - --define(GET_API, ["management/nodes", - ?PRINT("management/nodes/~s"), - "monitoring/nodes", - ?PRINT("monitoring/nodes/~s"), - "monitoring/listeners", - ?PRINT("monitoring/listeners/~s"), - "monitoring/metrics", - ?PRINT("monitoring/metrics/~s"), - "monitoring/stats", - ?PRINT("monitoring/stats/~s"), - ?PRINT("nodes/~s/clients"), - "routes"]). - -all() -> - [{group, protocol}, - {group, pubsub}, - {group, session}, - {group, broker}, - {group, metrics}, - {group, stats}, - {group, hook}, - {group, http}, - {group, alarms}, - {group, cli}, - {group, cleanSession}]. - -groups() -> - [{protocol, [sequence], - [mqtt_connect, - mqtt_ssl_twoway, - mqtt_ssl_oneway - ]}, - {pubsub, [sequence], - [subscribe_unsubscribe, - publish, pubsub, - t_local_subscribe, - t_shared_subscribe, - 'pubsub#', 'pubsub+']}, - {session, [sequence], - [start_session]}, - {broker, [sequence], - [hook_unhook]}, - {metrics, [sequence], - [inc_dec_metric]}, - {stats, [sequence], - [set_get_stat]}, - {hook, [sequence], - [add_delete_hook, - run_hooks]}, - {http, [sequence], - [request_status, - request_publish, - get_api_lists - % websocket_test - ]}, - {alarms, [sequence], - [set_alarms] - }, - {cli, [sequence], - [ctl_register_cmd, - cli_status, - cli_broker, - cli_clients, - cli_sessions, - cli_routes, - cli_topics, - cli_subscriptions, - cli_bridges, - cli_plugins, - {listeners, [sequence], - [cli_listeners, - conflict_listeners - ]}, - cli_vm]}, - {cleanSession, [sequence], - [cleanSession_validate]}]. - -init_per_suite(Config) -> - NewConfig = generate_config(), - lists:foreach(fun set_app_env/1, NewConfig), - application:ensure_all_started(?APP), - Config. - -end_per_suite(_Config) -> - emqttd:shutdown(). - -%%-------------------------------------------------------------------- -%% Protocol Test -%%-------------------------------------------------------------------- - -mqtt_connect(_) -> - %% Issue #599 - %% Empty clientId and clean_session = false - ?assertEqual(<<32,2,0,2>>, connect_broker_(<<16,12,0,4,77,81,84,84,4,0,0,90,0,0>>, 4)), - %% Empty clientId and clean_session = true - ?assertEqual(<<32,2,0,0>>, connect_broker_(<<16,12,0,4,77,81,84,84,4,2,0,90,0,0>>, 4)). - -connect_broker_(Packet, RecvSize) -> - {ok, Sock} = gen_tcp:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}]), - gen_tcp:send(Sock, Packet), - {ok, Data} = gen_tcp:recv(Sock, RecvSize, 3000), - gen_tcp:close(Sock), - Data. - -mqtt_ssl_oneway(_) -> - emqttd:stop(), - change_opts(ssl_oneway), - emqttd:start(), - timer:sleep(6000), - {ok, SslOneWay} = emqttc:start_link([{host, "localhost"}, - {port, 8883}, - {client_id, <<"ssloneway">>}, ssl]), - timer:sleep(100), - emqttc:subscribe(SslOneWay, <<"topic">>, qos1), - {ok, Pub} = emqttc:start_link([{host, "localhost"}, - {client_id, <<"pub">>}]), - emqttc:publish(Pub, <<"topic">>, <<"SSL oneWay test">>, [{qos, 1}]), - timer:sleep(100), - receive {publish, _Topic, RM} -> - ?assertEqual(<<"SSL oneWay test">>, RM) - after 1000 -> false - end, - timer:sleep(100), - emqttc:disconnect(SslOneWay), - emqttc:disconnect(Pub). - -mqtt_ssl_twoway(_) -> - emqttd:stop(), - change_opts(ssl_twoway), - emqttd:start(), - timer:sleep(6000), - ClientSSl = [{Key, local_path(["etc", File])} || - {Key, File} <- ?MQTT_SSL_CLIENT], - {ok, SslTwoWay} = emqttc:start_link([{host, "localhost"}, - {port, 8883}, - {client_id, <<"ssltwoway">>}, - {ssl, ClientSSl}]), - {ok, Sub} = emqttc:start_link([{host, "localhost"}, - {client_id, <<"sub">>}]), - emqttc:subscribe(Sub, <<"topic">>, qos1), - emqttc:publish(SslTwoWay, <<"topic">>, <<"ssl client pub message">>, [{qos, 1}]), - timer:sleep(10), - receive {publish, _Topic, RM} -> - ?assertEqual(<<"ssl client pub message">>, RM) - after 1000 -> false - end, - emqttc:disconnect(SslTwoWay), - emqttc:disconnect(Sub). - -%%-------------------------------------------------------------------- -%% PubSub Test -%%-------------------------------------------------------------------- - -subscribe_unsubscribe(_) -> - ok = emqttd:subscribe(<<"topic">>, <<"clientId">>), - ok = emqttd:subscribe(<<"topic/1">>, <<"clientId">>, [{qos, 1}]), - ok = emqttd:subscribe(<<"topic/2">>, <<"clientId">>, [{qos, 2}]), - ok = emqttd:unsubscribe(<<"topic">>, <<"clientId">>), - ok = emqttd:unsubscribe(<<"topic/1">>, <<"clientId">>), - ok = emqttd:unsubscribe(<<"topic/2">>, <<"clientId">>). - -publish(_) -> - Msg = emqttd_message:make(ct, <<"test/pubsub">>, <<"hello">>), - ok = emqttd:subscribe(<<"test/+">>), - timer:sleep(10), - emqttd:publish(Msg), - ?assert(receive {dispatch, <<"test/+">>, Msg} -> true after 5 -> false end). - -pubsub(_) -> - Self = self(), - ok = emqttd:subscribe(<<"a/b/c">>, Self, [{qos, 1}]), - ?assertMatch({error, _}, emqttd:subscribe(<<"a/b/c">>, Self, [{qos, 2}])), - timer:sleep(10), - [{Self, <<"a/b/c">>}] = ets:lookup(mqtt_subscription, Self), - [{<<"a/b/c">>, Self}] = ets:lookup(mqtt_subscriber, <<"a/b/c">>), - emqttd:publish(emqttd_message:make(ct, <<"a/b/c">>, <<"hello">>)), - ?assert(receive {dispatch, <<"a/b/c">>, _} -> true after 2 -> false end), - spawn(fun() -> - emqttd:subscribe(<<"a/b/c">>), - emqttd:subscribe(<<"c/d/e">>), - timer:sleep(10), - emqttd:unsubscribe(<<"a/b/c">>) - end), - timer:sleep(20), - emqttd:unsubscribe(<<"a/b/c">>). - -t_local_subscribe(_) -> - ok = emqttd:subscribe("$local/topic0"), - ok = emqttd:subscribe("$local/topic1", <<"x">>), - ok = emqttd:subscribe("$local/topic2", <<"x">>, [{qos, 2}]), - timer:sleep(10), - ?assertEqual([self()], emqttd:subscribers("$local/topic0")), - ?assertEqual([{<<"x">>, self()}], emqttd:subscribers("$local/topic1")), - ?assertEqual([{{<<"x">>, self()}, <<"$local/topic1">>, []}, - {{<<"x">>, self()}, <<"$local/topic2">>, [{qos,2}]}], - emqttd:subscriptions(<<"x">>)), - - ?assertEqual(ok, emqttd:unsubscribe("$local/topic0")), - ?assertMatch({error, {subscription_not_found, _}}, emqttd:unsubscribe("$local/topic0")), - ?assertEqual(ok, emqttd:unsubscribe("$local/topic1", <<"x">>)), - ?assertEqual(ok, emqttd:unsubscribe("$local/topic2", <<"x">>)), - ?assertEqual([], emqttd:subscribers("topic1")), - ?assertEqual([], emqttd:subscriptions(<<"x">>)). - -t_shared_subscribe(_) -> - emqttd:subscribe("$local/$share/group1/topic1"), - emqttd:subscribe("$share/group2/topic2"), - emqttd:subscribe("$queue/topic3"), - timer:sleep(10), - ?assertEqual([self()], emqttd:subscribers(<<"$local/$share/group1/topic1">>)), - ?assertEqual([{self(), <<"$local/$share/group1/topic1">>, []}, - {self(), <<"$queue/topic3">>, []}, - {self(), <<"$share/group2/topic2">>, []}], - lists:sort(emqttd:subscriptions(self()))), - emqttd:unsubscribe("$local/$share/group1/topic1"), - emqttd:unsubscribe("$share/group2/topic2"), - emqttd:unsubscribe("$queue/topic3"), - ?assertEqual([], lists:sort(emqttd:subscriptions(self()))). - -'pubsub#'(_) -> - emqttd:subscribe(<<"a/#">>), - timer:sleep(10), - emqttd:publish(emqttd_message:make(ct, <<"a/b/c">>, <<"hello">>)), - ?assert(receive {dispatch, <<"a/#">>, _} -> true after 2 -> false end), - emqttd:unsubscribe(<<"a/#">>). - -'pubsub+'(_) -> - emqttd:subscribe(<<"a/+/+">>), - timer:sleep(10), - emqttd:publish(emqttd_message:make(ct, <<"a/b/c">>, <<"hello">>)), - ?assert(receive {dispatch, <<"a/+/+">>, _} -> true after 1 -> false end), - emqttd:unsubscribe(<<"a/+/+">>). - -loop_recv(Topic, Timeout) -> - loop_recv(Topic, Timeout, []). - -loop_recv(Topic, Timeout, Acc) -> - receive - {dispatch, Topic, Msg} -> - loop_recv(Topic, Timeout, [Msg|Acc]) - after - Timeout -> {ok, Acc} - end. - -recv_loop(Msgs) -> - receive - {dispatch, _Topic, Msg} -> - recv_loop([Msg|Msgs]) - after - 100 -> lists:reverse(Msgs) - end. - -%%-------------------------------------------------------------------- -%% Session Group -%%-------------------------------------------------------------------- - -start_session(_) -> - {ok, ClientPid} = emqttd_mock_client:start_link(<<"clientId">>), - {ok, SessPid} = emqttd_mock_client:start_session(ClientPid), - Message = emqttd_message:make(<<"clientId">>, 2, <<"topic">>, <<"hello">>), - Message1 = Message#mqtt_message{pktid = 1}, - emqttd_session:publish(SessPid, Message1), - emqttd_session:pubrel(SessPid, 1), - emqttd_session:subscribe(SessPid, [{<<"topic/session">>, [{qos, 2}]}]), - Message2 = emqttd_message:make(<<"clientId">>, 1, <<"topic/session">>, <<"test">>), - emqttd_session:publish(SessPid, Message2), - emqttd_session:unsubscribe(SessPid, [{<<"topic/session">>, []}]), - emqttd_mock_client:stop(ClientPid). - -%%-------------------------------------------------------------------- -%% Broker Group -%%-------------------------------------------------------------------- -hook_unhook(_) -> - ok. - -%%-------------------------------------------------------------------- -%% Metric Group -%%-------------------------------------------------------------------- -inc_dec_metric(_) -> - emqttd_metrics:inc(gauge, 'messages/retained', 10), - emqttd_metrics:dec(gauge, 'messages/retained', 10). - -%%-------------------------------------------------------------------- -%% Stats Group -%%-------------------------------------------------------------------- -set_get_stat(_) -> - emqttd_stats:setstat('retained/max', 99), - 99 = emqttd_stats:getstat('retained/max'). - -%%-------------------------------------------------------------------- -%% Hook Test -%%-------------------------------------------------------------------- - -add_delete_hook(_) -> - ok = emqttd:hook(test_hook, fun ?MODULE:hook_fun1/1, []), - ok = emqttd:hook(test_hook, {tag, fun ?MODULE:hook_fun2/1}, []), - {error, already_hooked} = emqttd:hook(test_hook, {tag, fun ?MODULE:hook_fun2/1}, []), - Callbacks = [{callback, undefined, fun ?MODULE:hook_fun1/1, [], 0}, - {callback, tag, fun ?MODULE:hook_fun2/1, [], 0}], - Callbacks = emqttd_hooks:lookup(test_hook), - ok = emqttd:unhook(test_hook, fun ?MODULE:hook_fun1/1), - ct:print("Callbacks: ~p~n", [emqttd_hooks:lookup(test_hook)]), - ok = emqttd:unhook(test_hook, {tag, fun ?MODULE:hook_fun2/1}), - {error, not_found} = emqttd:unhook(test_hook1, {tag, fun ?MODULE:hook_fun2/1}), - [] = emqttd_hooks:lookup(test_hook), - - ok = emqttd:hook(emqttd_hook, fun ?MODULE:hook_fun1/1, [], 9), - ok = emqttd:hook(emqttd_hook, {"tag", fun ?MODULE:hook_fun2/1}, [], 8), - Callbacks2 = [{callback, "tag", fun ?MODULE:hook_fun2/1, [], 8}, - {callback, undefined, fun ?MODULE:hook_fun1/1, [], 9}], - Callbacks2 = emqttd_hooks:lookup(emqttd_hook), - ok = emqttd:unhook(emqttd_hook, fun ?MODULE:hook_fun1/1), - ok = emqttd:unhook(emqttd_hook, {"tag", fun ?MODULE:hook_fun2/1}), - [] = emqttd_hooks:lookup(emqttd_hook). - -run_hooks(_) -> - ok = emqttd:hook(foldl_hook, fun ?MODULE:hook_fun3/4, [init]), - ok = emqttd:hook(foldl_hook, {tag, fun ?MODULE:hook_fun3/4}, [init]), - ok = emqttd:hook(foldl_hook, fun ?MODULE:hook_fun4/4, [init]), - ok = emqttd:hook(foldl_hook, fun ?MODULE:hook_fun5/4, [init]), - {stop, [r3, r2]} = emqttd:run_hooks(foldl_hook, [arg1, arg2], []), - {ok, []} = emqttd:run_hooks(unknown_hook, [], []), - - ok = emqttd:hook(foreach_hook, fun ?MODULE:hook_fun6/2, [initArg]), - ok = emqttd:hook(foreach_hook, {tag, fun ?MODULE:hook_fun6/2}, [initArg]), - ok = emqttd:hook(foreach_hook, fun ?MODULE:hook_fun7/2, [initArg]), - ok = emqttd:hook(foreach_hook, fun ?MODULE:hook_fun8/2, [initArg]), - stop = emqttd:run_hooks(foreach_hook, [arg]). - -hook_fun1([]) -> ok. -hook_fun2([]) -> {ok, []}. - -hook_fun3(arg1, arg2, _Acc, init) -> ok. -hook_fun4(arg1, arg2, Acc, init) -> {ok, [r2 | Acc]}. -hook_fun5(arg1, arg2, Acc, init) -> {stop, [r3 | Acc]}. - -hook_fun6(arg, initArg) -> ok. -hook_fun7(arg, initArg) -> any. -hook_fun8(arg, initArg) -> stop. - -%%-------------------------------------------------------------------- -%% HTTP Request Test -%%-------------------------------------------------------------------- - -request_status(_) -> - {InternalStatus, _ProvidedStatus} = init:get_status(), - AppStatus = - case lists:keysearch(?APP, 1, application:which_applications()) of - false -> not_running; - {value, _Val} -> running - end, - Status = iolist_to_binary(io_lib:format("Node ~s is ~s~nemqttd is ~s", - [node(), InternalStatus, AppStatus])), - Url = "http://127.0.0.1:8080/status", - {ok, {{"HTTP/1.1", 200, "OK"}, _, Return}} = - httpc:request(get, {Url, []}, [], []), - ?assertEqual(binary_to_list(Status), Return). - -request_publish(_) -> - application:start(emq_dashboard), - emqttc:start_link([{host, "localhost"}, - {port, 1883}, - {client_id, <<"random">>}, - {clean_sess, false}]), - SubParams = "{\"qos\":1, \"topic\" : \"a\/b\/c\", \"client_id\" :\"random\"}", - ?assert(connect_emqttd_pubsub_(post, "api/v2/mqtt/subscribe", SubParams, auth_header_("admin", "public"))), - ok = emqttd:subscribe(<<"a/b/c">>, self(), [{qos, 1}]), - Params = "{\"qos\":1, \"retain\":false, \"topic\" : \"a\/b\/c\", \"messages\" :\"hello\"}", - ?assert(connect_emqttd_pubsub_(post, "api/v2/mqtt/publish", Params, auth_header_("admin", "public"))), - ?assert(receive {dispatch, <<"a/b/c">>, _} -> true after 2 -> false end), - - UnSubParams = "{\"topic\" : \"a\/b\/c\", \"client_id\" :\"random\"}", - ?assert(connect_emqttd_pubsub_(post, "api/v2/mqtt/unsubscribe", UnSubParams, auth_header_("admin", "public"))). - -connect_emqttd_pubsub_(Method, Api, Params, Auth) -> - Url = "http://127.0.0.1:8080/" ++ Api, - case httpc:request(Method, {Url, [Auth], ?CONTENT_TYPE, Params}, [], []) of - {error, socket_closed_remotely} -> - false; - {ok, {{"HTTP/1.1", 200, "OK"}, _, _Return} } -> - true; - {ok, {{"HTTP/1.1", 400, _}, _, []}} -> - false; - {ok, {{"HTTP/1.1", 404, _}, _, []}} -> - false - end. - -auth_header_(User, Pass) -> - Encoded = base64:encode_to_string(lists:append([User,":",Pass])), - {"Authorization","Basic " ++ Encoded}. - -get_api_lists(_Config) -> - lists:foreach(fun request/1, ?GET_API). - -websocket_test(_) -> - Conn = esockd_connection:new(esockd_transport, nil, []), - Req = mochiweb_request:new(Conn, 'GET', "/mqtt", {1, 1}, - mochiweb_headers:make([{"Sec-WebSocket-Key","Xn3fdKyc3qEXPuj2A3O+ZA=="}])), - - ct:log("Req:~p", [Req]), - emqttd_http:handle_request(Req). - -set_alarms(_) -> - AlarmTest = #mqtt_alarm{id = <<"1">>, severity = error, title="alarm title", summary="alarm summary"}, - emqttd_alarm:set_alarm(AlarmTest), - Alarms = emqttd_alarm:get_alarms(), - ?assertEqual(1, length(Alarms)), - emqttd_alarm:clear_alarm(<<"1">>), - [] = emqttd_alarm:get_alarms(). - -%%-------------------------------------------------------------------- -%% Cli group -%%-------------------------------------------------------------------- - -ctl_register_cmd(_) -> - emqttd_ctl:register_cmd(test_cmd, {?MODULE, test_cmd}), - erlang:yield(), - timer:sleep(5), - [{?MODULE, test_cmd}] = emqttd_ctl:lookup(test_cmd), - emqttd_ctl:run(["test_cmd", "arg1", "arg2"]), - emqttd_ctl:unregister_cmd(test_cmd). - -test_cmd(["arg1", "arg2"]) -> - ct:print("test_cmd is called"); - -test_cmd([]) -> - io:format("test command"). - -cli_status(_) -> - emqttd_cli:status([]). - -cli_broker(_) -> - emqttd_cli:broker([]), - emqttd_cli:broker(["stats"]), - emqttd_cli:broker(["metrics"]), - emqttd_cli:broker(["pubsub"]). - -cli_clients(_) -> - emqttd_cli:clients(["list"]), - emqttd_cli:clients(["show", "clientId"]), - emqttd_cli:clients(["kick", "clientId"]). - -cli_sessions(_) -> - emqttd_cli:sessions(["list"]), - emqttd_cli:sessions(["list", "persistent"]), - emqttd_cli:sessions(["list", "transient"]), - emqttd_cli:sessions(["show", "clientId"]). - -cli_routes(_) -> - emqttd:subscribe(<<"topic/route">>), - emqttd_cli:routes(["list"]), - emqttd_cli:routes(["show", "topic/route"]), - emqttd:unsubscribe(<<"topic/route">>). - -cli_topics(_) -> - emqttd:subscribe(<<"topic">>), - emqttd_cli:topics(["list"]), - emqttd_cli:topics(["show", "topic"]), - emqttd:unsubscribe(<<"topic">>). - -cli_subscriptions(_) -> - emqttd_cli:subscriptions(["list"]), - emqttd_cli:subscriptions(["show", "clientId"]), - emqttd_cli:subscriptions(["add", "clientId", "topic", "2"]), - emqttd_cli:subscriptions(["del", "clientId", "topic"]). - -cli_plugins(_) -> - emqttd_cli:plugins(["list"]), - emqttd_cli:plugins(["load", "emqttd_plugin_template"]), - emqttd_cli:plugins(["unload", "emqttd_plugin_template"]). - -cli_bridges(_) -> - emqttd_cli:bridges(["list"]), - emqttd_cli:bridges(["start", "a@127.0.0.1", "topic"]), - emqttd_cli:bridges(["stop", "a@127.0.0.1", "topic"]). - -cli_listeners(_) -> - emqttd_cli:listeners([]). - -conflict_listeners(_) -> - F = - fun() -> - process_flag(trap_exit, true), - emqttc:start_link([{host, "localhost"}, - {port, 1883}, - {client_id, <<"c1">>}, - {clean_sess, false}]) - end, - spawn_link(F), - - {ok, C2} = emqttc:start_link([{host, "localhost"}, - {port, 1883}, - {client_id, <<"c1">>}, - {clean_sess, false}]), - timer:sleep(100), - - Listeners = - lists:map(fun({{Protocol, ListenOn}, Pid}) -> - Key = atom_to_list(Protocol) ++ ":" ++ esockd:to_string(ListenOn), - {Key, [{acceptors, esockd:get_acceptors(Pid)}, - {max_clients, esockd:get_max_clients(Pid)}, - {current_clients, esockd:get_current_clients(Pid)}, - {shutdown_count, esockd:get_shutdown_count(Pid)}]} - end, esockd:listeners()), - L = proplists:get_value("mqtt:tcp:0.0.0.0:1883", Listeners), - ?assertEqual(1, proplists:get_value(current_clients, L)), - ?assertEqual(1, proplists:get_value(conflict, proplists:get_value(shutdown_count, L))), - timer:sleep(100), - emqttc:disconnect(C2). - -cli_vm(_) -> - emqttd_cli:vm([]), - emqttd_cli:vm(["ports"]). - -cleanSession_validate(_) -> - {ok, C1} = emqttc:start_link([{host, "localhost"}, - {port, 1883}, - {client_id, <<"c1">>}, - {clean_sess, false}]), - timer:sleep(10), - emqttc:subscribe(C1, <<"topic">>, qos0), - emqttc:disconnect(C1), - {ok, Pub} = emqttc:start_link([{host, "localhost"}, - {port, 1883}, - {client_id, <<"pub">>}]), - - emqttc:publish(Pub, <<"topic">>, <<"m1">>, [{qos, 0}]), - timer:sleep(10), - {ok, C11} = emqttc:start_link([{host, "localhost"}, - {port, 1883}, - {client_id, <<"c1">>}, - {clean_sess, false}]), - timer:sleep(100), - Metrics = emqttd_metrics:all(), - ?assertEqual(1, proplists:get_value('messages/qos0/sent', Metrics)), - ?assertEqual(1, proplists:get_value('messages/qos0/received', Metrics)), - emqttc:disconnect(Pub), - emqttc:disconnect(C11). - -change_opts(SslType) -> - {ok, Listeners} = application:get_env(?APP, listeners), - NewListeners = - lists:foldl(fun({Protocol, Port, Opts} = Listener, Acc) -> - case Protocol of - ssl -> - SslOpts = proplists:get_value(sslopts, Opts), - Keyfile = local_path(["etc/certs", "key.pem"]), - Certfile = local_path(["etc/certs", "cert.pem"]), - TupleList1 = lists:keyreplace(keyfile, 1, SslOpts, {keyfile, Keyfile}), - TupleList2 = lists:keyreplace(certfile, 1, TupleList1, {certfile, Certfile}), - TupleList3 = - case SslType of - ssl_twoway-> - CAfile = local_path(["etc", proplists:get_value(cacertfile, ?MQTT_SSL_TWOWAY)]), - MutSslList = lists:keyreplace(cacertfile, 1, ?MQTT_SSL_TWOWAY, {cacertfile, CAfile}), - lists:merge(TupleList2, MutSslList); - _ -> - lists:filter(fun ({cacertfile, _}) -> false; - ({verify, _}) -> false; - ({fail_if_no_peer_cert, _}) -> false; - (_) -> true - end, TupleList2) - end, - [{Protocol, Port, lists:keyreplace(sslopts, 1, Opts, {sslopts, TupleList3})} | Acc]; - _ -> - [Listener | Acc] - end - end, [], Listeners), - application:set_env(?APP, listeners, NewListeners). - -generate_config() -> - Schema = cuttlefish_schema:files([local_path(["priv", "emq.schema"])]), - Conf = conf_parse:file([local_path(["etc", "emq.conf"])]), - cuttlefish_generator:map(Schema, Conf). - -get_base_dir(Module) -> - {file, Here} = code:is_loaded(Module), - filename:dirname(filename:dirname(Here)). - -get_base_dir() -> - get_base_dir(?MODULE). - -local_path(Components, Module) -> - filename:join([get_base_dir(Module) | Components]). - -local_path(Components) -> - local_path(Components, ?MODULE). - -set_app_env({App, Lists}) -> - lists:foreach(fun({Par, Var}) -> - application:set_env(App, Par, Var) - end, Lists). - -request(Path) -> - http_get(get, Path). - -http_get(Method, Path) -> - req(Method, Path, []). - -http_put(Method, Path, Params) -> - req(Method, Path, format_for_upload(Params)). - -http_post(Method, Path, Params) -> - req(Method, Path, format_for_upload(Params)). - -req(Method, Path, Body) -> - Url = ?URL ++ Path, - Headers = auth_header_("admin", "public"), - case httpc:request(Method, {Url, [Headers]}, [], []) of - {error, socket_closed_remotely} -> - false; - {ok, {{"HTTP/1.1", 200, "OK"}, _, _Return} } -> - true; - {ok, {{"HTTP/1.1", 400, _}, _, []}} -> - false; - {ok, {{"HTTP/1.1", 404, _}, _, []}} -> - false - end. - -format_for_upload(none) -> - <<"">>; -format_for_upload(List) -> - iolist_to_binary(mochijson2:encode(List)). diff --git a/test/emqttd_access_SUITE.erl b/test/emqttd_access_SUITE.erl deleted file mode 100644 index c3529d935..000000000 --- a/test/emqttd_access_SUITE.erl +++ /dev/null @@ -1,179 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_access_SUITE). - --compile(export_all). - --include("emqttd.hrl"). - --define(AC, emqttd_access_control). - --import(emqttd_access_rule, [compile/1, match/3]). - -all() -> - [{group, access_control}, - {group, access_rule}]. - -groups() -> - [{access_control, [sequence], - [reload_acl, - register_mod, - unregister_mod, - check_acl]}, - {access_rule, [], - [compile_rule, - match_rule]}]. - -init_per_group(access_control, Config) -> - application:load(emqttd), - prepare_config(), - Config; - -init_per_group(_Group, Config) -> - Config. - -prepare_config() -> - Rules = [{allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]}, - {allow, {user, "testuser"}, subscribe, ["a/b/c", "d/e/f/#"]}, - {allow, {user, "admin"}, pubsub, ["a/b/c", "d/e/f/#"]}, - {allow, {client, "testClient"}, subscribe, ["testTopics/testClient"]}, - {allow, all, subscribe, ["clients/%c"]}, - {allow, all, pubsub, ["users/%u/#"]}, - {deny, all, subscribe, ["$SYS/#", "#"]}, - {deny, all}], - write_config("access_SUITE_acl.conf", Rules), - Config = [{auth, anonymous, []}, - {acl, internal, [{config, "access_SUITE_acl.conf"}, - {nomatch, allow}]}], - write_config("access_SUITE_emqttd.conf", Config), - application:set_env(emqttd, conf, "access_SUITE_emqttd.conf"). - -write_config(Filename, Terms) -> - file:write_file(Filename, [io_lib:format("~tp.~n", [Term]) || Term <- Terms]). - -end_per_group(_Group, Config) -> - Config. - -init_per_testcase(TestCase, Config) when TestCase =:= reload_acl; - TestCase =:= register_mod; - TestCase =:= unregister_mod; - TestCase =:= check_acl -> - {ok, _Pid} = ?AC:start_link(), Config; - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(TestCase, _Config) when TestCase =:= reload_acl; - TestCase =:= register_mod; - TestCase =:= unregister_mod; - TestCase =:= check_acl -> - ?AC:stop(); - -end_per_testcase(_TestCase, _Config) -> - ok. - -%%-------------------------------------------------------------------- -%% emqttd_access_control -%%-------------------------------------------------------------------- - -reload_acl(_) -> - [] = ?AC:reload_acl(). - -register_mod(_) -> - ok = ?AC:register_mod(acl, emqttd_acl_test_mod, []), - {error, already_existed} = ?AC:register_mod(acl, emqttd_acl_test_mod, []), - [{emqttd_acl_test_mod, _, 0}] = ?AC:lookup_mods(acl), - ok = ?AC:register_mod(auth, emqttd_auth_anonymous_test_mod,[]), - ok = ?AC:register_mod(auth, emqttd_auth_dashboard, [], 99), - [{emqttd_auth_dashboard, _, 99}, - {emqttd_auth_anonymous_test_mod, _, 0}] = ?AC:lookup_mods(auth). - -unregister_mod(_) -> - ok = ?AC:register_mod(acl, emqttd_acl_test_mod, []), - [{emqttd_acl_test_mod, _, 0}] = ?AC:lookup_mods(acl), - ok = ?AC:unregister_mod(acl, emqttd_acl_test_mod), - timer:sleep(5), - [] = ?AC:lookup_mods(acl), - ok = ?AC:register_mod(auth, emqttd_auth_anonymous_test_mod,[]), - [{emqttd_auth_anonymous_test_mod, _, 0}] = ?AC:lookup_mods(auth), - - ok = ?AC:unregister_mod(auth, emqttd_auth_anonymous_test_mod), - timer:sleep(5), - [] = ?AC:lookup_mods(auth). - -check_acl(_) -> - User1 = #mqtt_client{client_id = <<"client1">>, username = <<"testuser">>}, - User2 = #mqtt_client{client_id = <<"client2">>, username = <<"xyz">>}, - allow = ?AC:check_acl(User1, subscribe, <<"users/testuser/1">>), - allow = ?AC:check_acl(User1, subscribe, <<"clients/client1">>), - allow = ?AC:check_acl(User1, subscribe, <<"clients/client1/x/y">>), - allow = ?AC:check_acl(User1, publish, <<"users/testuser/1">>), - allow = ?AC:check_acl(User1, subscribe, <<"a/b/c">>), - allow = ?AC:check_acl(User2, subscribe, <<"a/b/c">>). - -%%-------------------------------------------------------------------- -%% emqttd_access_rule -%%-------------------------------------------------------------------- - -compile_rule(_) -> - - {allow, {'and', [{ipaddr, {{127,0,0,1}, {127,0,0,1}, 32}}, - {user, <<"user">>}]}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]} = - compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"user">>}]}, subscribe, ["$SYS/#", "#"]}), - {allow, {'or', [{ipaddr, {{127,0,0,1}, {127,0,0,1}, 32}}, - {user, <<"user">>}]}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]} = - compile({allow, {'or', [{ipaddr, "127.0.0.1"}, {user, <<"user">>}]}, subscribe, ["$SYS/#", "#"]}), - - {allow, {ipaddr, {{127,0,0,1}, {127,0,0,1}, 32}}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]} = - compile({allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]}), - {allow, {user, <<"testuser">>}, subscribe, [ [<<"a">>, <<"b">>, <<"c">>], [<<"d">>, <<"e">>, <<"f">>, '#'] ]} = - compile({allow, {user, "testuser"}, subscribe, ["a/b/c", "d/e/f/#"]}), - {allow, {user, <<"admin">>}, pubsub, [ [<<"d">>, <<"e">>, <<"f">>, '#'] ]} = - compile({allow, {user, "admin"}, pubsub, ["d/e/f/#"]}), - {allow, {client, <<"testClient">>}, publish, [ [<<"testTopics">>, <<"testClient">>] ]} = - compile({allow, {client, "testClient"}, publish, ["testTopics/testClient"]}), - {allow, all, pubsub, [{pattern, [<<"clients">>, <<"%c">>]}]} = - compile({allow, all, pubsub, ["clients/%c"]}), - {allow, all, subscribe, [{pattern, [<<"users">>, <<"%u">>, '#']}]} = - compile({allow, all, subscribe, ["users/%u/#"]}), - {deny, all, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]} = - compile({deny, all, subscribe, ["$SYS/#", "#"]}), - {allow, all} = compile({allow, all}), - {deny, all} = compile({deny, all}). - -match_rule(_) -> - User = #mqtt_client{peername = {{127,0,0,1}, 2948}, client_id = <<"testClient">>, username = <<"TestUser">>}, - User2 = #mqtt_client{peername = {{192,168,0,10}, 3028}, client_id = <<"testClient">>, username = <<"TestUser">>}, - - {matched, allow} = match(User, <<"Test/Topic">>, {allow, all}), - {matched, deny} = match(User, <<"Test/Topic">>, {deny, all}), - {matched, allow} = match(User, <<"Test/Topic">>, compile({allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]})), - {matched, allow} = match(User2, <<"Test/Topic">>, compile({allow, {ipaddr, "192.168.0.1/24"}, subscribe, ["$SYS/#", "#"]})), - {matched, allow} = match(User, <<"d/e/f/x">>, compile({allow, {user, "TestUser"}, subscribe, ["a/b/c", "d/e/f/#"]})), - nomatch = match(User, <<"d/e/f/x">>, compile({allow, {user, "admin"}, pubsub, ["d/e/f/#"]})), - {matched, allow} = match(User, <<"testTopics/testClient">>, compile({allow, {client, "testClient"}, publish, ["testTopics/testClient"]})), - {matched, allow} = match(User, <<"clients/testClient">>, compile({allow, all, pubsub, ["clients/%c"]})), - {matched, allow} = match(#mqtt_client{username = <<"user2">>}, <<"users/user2/abc/def">>, - compile({allow, all, subscribe, ["users/%u/#"]})), - {matched, deny} = match(User, <<"d/e/f">>, compile({deny, all, subscribe, ["$SYS/#", "#"]})), - Rule = compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"WrongUser">>}]}, publish, <<"Topic">>}), - nomatch = match(User, <<"Topic">>, Rule), - AndRule = compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"TestUser">>}]}, publish, <<"Topic">>}), - {matched, allow} = match(User, <<"Topic">>, AndRule), - OrRule = compile({allow, {'or', [{ipaddr, "127.0.0.1"}, {user, <<"WrongUser">>}]}, publish, ["Topic"]}), - {matched, allow} = match(User, <<"Topic">>, OrRule). - diff --git a/test/emqttd_config_SUITE.erl b/test/emqttd_config_SUITE.erl deleted file mode 100644 index 8b227b1b2..000000000 --- a/test/emqttd_config_SUITE.erl +++ /dev/null @@ -1,149 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_config_SUITE). - --compile(export_all). - --include("emqttd.hrl"). - --include_lib("eunit/include/eunit.hrl"). - --include_lib("common_test/include/ct.hrl"). - -all() -> - [{group, emq_config}]. - -groups() -> - [{emq_config, [sequence], - [run_protocol_cmd, - run_client_cmd, - run_session_cmd, - run_queue_cmd, - run_auth_cmd, - run_lager_cmd, - run_connection_cmd, - run_broker_config] - }]. - -init_per_suite(Config) -> - Config. - -end_per_suite(Config) -> - Config. - -run_protocol_cmd(_Config) -> - SetConfigKeys = [{"max_clientid_len=2048", int}, - {"max_packet_size=1024", int}, - % {"websocket_protocol_header=off", atom}, - {"keepalive_backoff=0.5", float}], - lists:foreach(fun set_cmd/1, SetConfigKeys), - R = lists:sort(lists:map(fun env_value/1, SetConfigKeys)), - {ok, E} = application:get_env(emqttd, protocol), - ?assertEqual(R, lists:sort(E)), - emqttd_cli_config:run(["config", "set", "mqtt.websocket_protocol_header=off", "--app=emqttd"]), - {ok, E1} = application:get_env(emqttd, websocket_protocol_header), - ?assertEqual(false, E1). - -run_client_cmd(_Config) -> - SetConfigKeys = [{"max_publish_rate=100", int}, - {"idle_timeout=60s", date}, - {"enable_stats=on", atom}], - lists:foreach(fun(Key) -> set_cmd("client", Key) end, SetConfigKeys), - R = lists:sort(lists:map(fun(Key) -> env_value("client", Key) end, SetConfigKeys)), - {ok, E} = application:get_env(emqttd, client), - ?assertEqual(R, lists:sort(E)). - -run_session_cmd(_Config) -> - SetConfigKeys = [{"max_subscriptions=5", int}, - {"upgrade_qos=on", atom}, - {"max_inflight=64", int}, - {"retry_interval=60s", date}, - {"max_awaiting_rel=200", int}, - {"await_rel_timeout=60s",date}, - {"enable_stats=on", atom}, - {"expiry_interval=60s", date}, - {"ignore_loop_deliver=true", atom}], - lists:foreach(fun(Key) -> set_cmd("session", Key) end, SetConfigKeys), - R = lists:sort(lists:map(fun env_value/1, SetConfigKeys)), - {ok, E} = application:get_env(emqttd, session), - ?assertEqual(R, lists:sort(E)). - -run_queue_cmd(_Config) -> - SetConfigKeys = [{"type=priority", atom}, - {"priority=hah", string}, - {"max_length=2000", int}, - {"low_watermark=40%",percent}, - {"high_watermark=80%", percent}, - {"store_qos0=false", atom}], - lists:foreach(fun(Key) -> set_cmd("mqueue", Key) end, SetConfigKeys), - R = lists:sort(lists:map(fun env_value/1, SetConfigKeys)), - {ok, E} = application:get_env(emqttd, mqueue), - ?assertEqual(R, lists:sort(E)). - -run_auth_cmd(_Config) -> - SetConfigKeys = [{"allow_anonymous=true", atom}, - {"acl_nomatch=deny", atom}, - {"acl_file=etc/test.acl", string}, - {"cache_acl=false", atom}], - lists:foreach(fun set_cmd/1, SetConfigKeys), - {ok, true} = application:get_env(emqttd, allow_anonymous), - {ok, deny} = application:get_env(emqttd, acl_nomatch), - {ok, "etc/test.acl"} = application:get_env(emqttd, acl_file), - {ok, false} = application:get_env(emqttd, cache_acl). - -run_lager_cmd(_Config) -> - emqttd_cli_config:run(["config", "set", "log.console.level=info", "--app=emqttd"]), - ok. - -run_connection_cmd(_Config) -> - emqttd_cli_config:run(["config", "set", "mqtt.conn.force_gc_count=1000", "--app=emqttd"]), - {ok, E} = application:get_env(emqttd, conn_force_gc_count), - ?assertEqual(1000, E). - -run_broker_config(_Config) -> - emqttd_cli_config:run(["config", "set", "mqtt.broker.sys_interval=6000ms", "--app=emqttd"]), - {ok, E} = application:get_env(emqttd, broker_sys_interval), - ?assertEqual(6000, E). - -env_value("client", {Key, Type}) -> - case string:split(Key, "=") of - ["max_publish_rate", V] -> - {list_to_atom("max_publish_rate"), format(Type, V)}; - [K, V] -> - {list_to_atom(string:join(["client", K], "_")), format(Type, V)} - end. - -env_value({Key, Type}) -> - [K, V] = string:split(Key, "="), - {list_to_atom(K), format(Type, V)}. - -format(string, S) -> S; -format(atom, "on") -> true; -format(atom, "off") -> false; -format(atom, A) -> list_to_atom(A); -format(float, F) -> list_to_float(F); -format(percent, P) -> - {match, [N]} = re:run(P, "^([0-9]+)%$", [{capture, all_but_first, list}]), - list_to_integer(N) / 100; -format(int, I) -> list_to_integer(I); -format(date, _I) -> 60000. - -set_cmd({Key, _Type}) -> - emqttd_cli_config:run(["config", "set", string:join(["mqtt", Key], "."), "--app=emqttd"]). - -set_cmd(Pre, {Key, _Type}) -> - emqttd_cli_config:run(["config", "set", string:join(["mqtt", Pre, Key], "."), "--app=emqttd"]). diff --git a/test/emqttd_inflight_SUITE.erl b/test/emqttd_inflight_SUITE.erl deleted file mode 100644 index d3800fc72..000000000 --- a/test/emqttd_inflight_SUITE.erl +++ /dev/null @@ -1,51 +0,0 @@ -%% -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% - --module(emqttd_inflight_SUITE). - --author("Feng Lee "). - --include_lib("eunit/include/eunit.hrl"). - -%% CT --compile(export_all). - -all() -> [t_contain, t_lookup, t_insert, t_update, t_delete, t_window, - t_is_full, t_is_empty]. - -t_contain(_) -> - Inflight = emqttd_inflight:new(0), - ?assertNot(Inflight:contain(k)), - Inflight1 = Inflight:insert(k, v), - ?assert(Inflight1:contain(k)). - -t_lookup(_) -> - Inflight = (emqttd_inflight:new(0)):insert(k, v), - ?assertEqual(v, Inflight:lookup(k)). - -t_insert(_) -> - Inflight = ((emqttd_inflight:new(0)):insert(k1, v1)):insert(k2, v2), - ?assertEqual(v2, Inflight:lookup(k2)). - -t_update(_) -> - Inflight = ((emqttd_inflight:new(0)):insert(k, v1)):update(k, v2), - ?assertEqual(v2, Inflight:lookup(k)). - -t_delete(_) -> - Inflight = ((emqttd_inflight:new(0)):insert(k, v1)):delete(k), - ?assert(Inflight:is_empty()). - -t_window(_) -> - ?assertEqual([], (emqttd_inflight:new(10)):window()), - Inflight = ((emqttd_inflight:new(0)):insert(1, 1)):insert(2, 2), - ?assertEqual([1, 2], Inflight:window()). - -t_is_full(_) -> - Inflight = ((emqttd_inflight:new(1)):insert(k, v1)), - ?assert(Inflight:is_full()). - -t_is_empty(_) -> - Inflight = ((emqttd_inflight:new(1)):insert(k, v1)), - ?assertNot(Inflight:is_empty()). - diff --git a/test/emqttd_mock_client.erl b/test/emqttd_mock_client.erl deleted file mode 100644 index f4d26fa30..000000000 --- a/test/emqttd_mock_client.erl +++ /dev/null @@ -1,63 +0,0 @@ - --module(emqttd_mock_client). - --behaviour(gen_server). - -%% ------------------------------------------------------------------ -%% API Function Exports -%% ------------------------------------------------------------------ - --export([start_link/1, start_session/1, stop/1]). - -%% ------------------------------------------------------------------ -%% gen_server Function Exports -%% ------------------------------------------------------------------ - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {clientid, session}). - -%% ------------------------------------------------------------------ -%% API Function Definitions -%% ------------------------------------------------------------------ - -start_link(ClientId) -> - gen_server:start_link(?MODULE, [ClientId], []). - -start_session(CPid) -> - gen_server:call(CPid, start_session). - -stop(CPid) -> - gen_server:call(CPid, stop). - -%% ------------------------------------------------------------------ -%% gen_server Function Definitions -%% ------------------------------------------------------------------ - -init([ClientId]) -> - {ok, #state{clientid = ClientId}}. - -handle_call(start_session, _From, State = #state{clientid = ClientId}) -> - {ok, SessPid, _} = emqttd_sm:start_session(true, {ClientId, undefined}), - {reply, {ok, SessPid}, State#state{session = SessPid}}; - -handle_call(stop, _From, State) -> - {stop, normal, ok, State}; - -handle_call(_Request, _From, State) -> - {reply, ok, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - - diff --git a/test/emqttd_mqueue_SUITE.erl b/test/emqttd_mqueue_SUITE.erl deleted file mode 100644 index f709f1478..000000000 --- a/test/emqttd_mqueue_SUITE.erl +++ /dev/null @@ -1,157 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_mqueue_SUITE). - --compile(export_all). - --include("emqttd.hrl"). - --define(Q, emqttd_mqueue). - -all() -> [t_in, t_in_qos0, t_out, t_simple_mqueue, t_priority_mqueue, - t_priority_mqueue2, t_infinity_priority_mqueue, - t_infinity_simple_mqueue]. - -t_in(_) -> - Opts = [{max_length, 5}, - {store_qos0, true}], - Q = ?Q:new(<<"testQ">>, Opts, alarm_fun()), - true = ?Q:is_empty(Q), - Q1 = ?Q:in(#mqtt_message{}, Q), - 1 = ?Q:len(Q1), - Q2 = ?Q:in(#mqtt_message{qos = 1}, Q1), - 2 = ?Q:len(Q2), - Q3 = ?Q:in(#mqtt_message{qos = 2}, Q2), - Q4 = ?Q:in(#mqtt_message{}, Q3), - Q5 = ?Q:in(#mqtt_message{}, Q4), - 5 = ?Q:len(Q5). - -t_in_qos0(_) -> - Opts = [{max_length, 5}, - {store_qos0, false}], - Q = ?Q:new(<<"testQ">>, Opts, alarm_fun()), - Q1 = ?Q:in(#mqtt_message{}, Q), - true = ?Q:is_empty(Q1), - Q2 = ?Q:in(#mqtt_message{qos = 0}, Q1), - true = ?Q:is_empty(Q2). - -t_out(_) -> - Opts = [{max_length, 5}, - {store_qos0, true}], - Q = ?Q:new(<<"testQ">>, Opts, alarm_fun()), - {empty, Q} = ?Q:out(Q), - Q1 = ?Q:in(#mqtt_message{}, Q), - {Value, Q2} = ?Q:out(Q1), - 0 = ?Q:len(Q2), - {value, #mqtt_message{}} = Value. - -t_simple_mqueue(_) -> - Opts = [{type, simple}, - {max_length, 3}, - {low_watermark, 0.2}, - {high_watermark, 0.6}, - {store_qos0, false}], - Q = ?Q:new("simple_queue", Opts, alarm_fun()), - simple = ?Q:type(Q), - 3 = ?Q:max_len(Q), - <<"simple_queue">> = ?Q:name(Q), - true = ?Q:is_empty(Q), - Q1 = ?Q:in(#mqtt_message{qos = 1, payload = <<"1">>}, Q), - Q2 = ?Q:in(#mqtt_message{qos = 1, payload = <<"2">>}, Q1), - Q3 = ?Q:in(#mqtt_message{qos = 1, payload = <<"3">>}, Q2), - Q4 = ?Q:in(#mqtt_message{qos = 1, payload = <<"4">>}, Q3), - 3 = ?Q:len(Q4), - {{value, Msg}, Q5} = ?Q:out(Q4), - <<"2">> = Msg#mqtt_message.payload, - [{len, 2}, {max_len, 3}, {dropped, 1}] = ?Q:stats(Q5). - -t_infinity_simple_mqueue(_) -> - Opts = [{type, simple}, - {max_length, 0}, - {low_watermark, 0.2}, - {high_watermark, 0.6}, - {store_qos0, false}], - Q = ?Q:new("infinity_simple_queue", Opts, alarm_fun()), - true = ?Q:is_empty(Q), - 0 = ?Q:max_len(Q), - Qx = lists:foldl(fun(I, AccQ) -> - ?Q:in(#mqtt_message{qos = 1, payload = iolist_to_binary([I])}, AccQ) - end, Q, lists:seq(1, 255)), - 255 = ?Q:len(Qx), - [{len, 255}, {max_len, 0}, {dropped, 0}] = ?Q:stats(Qx), - {{value, V}, _Qy} = ?Q:out(Qx), - <<1>> = V#mqtt_message.payload. - -t_priority_mqueue(_) -> - Opts = [{type, priority}, - {priority, [{<<"t">>, 10}]}, - {max_length, 3}, - {low_watermark, 0.2}, - {high_watermark, 0.6}, - {store_qos0, false}], - Q = ?Q:new("priority_queue", Opts, alarm_fun()), - priority = ?Q:type(Q), - 3 = ?Q:max_len(Q), - <<"priority_queue">> = ?Q:name(Q), - - true = ?Q:is_empty(Q), - Q1 = ?Q:in(#mqtt_message{qos = 1, topic = <<"t1">>}, Q), - Q2 = ?Q:in(#mqtt_message{qos = 1, topic = <<"t">>}, Q1), - Q3 = ?Q:in(#mqtt_message{qos = 1, topic = <<"t2">>}, Q2), - 3 = ?Q:len(Q3), - Q4 = ?Q:in(#mqtt_message{qos = 1, topic = <<"t1">>}, Q3), - 4 = ?Q:len(Q4), - Q5 = ?Q:in(#mqtt_message{qos = 1, topic = <<"t1">>}, Q4), - 5 = ?Q:len(Q5), - Q6 = ?Q:in(#mqtt_message{qos = 1, topic = <<"t1">>}, Q5), - 5 = ?Q:len(Q6), - {{value, Msg}, _Q7} = ?Q:out(Q6), - <<"t">> = Msg#mqtt_message.topic. - -t_infinity_priority_mqueue(_) -> - Opts = [{type, priority}, - {priority, [{<<"t1">>, 10}, {<<"t2">>, 8}]}, - {max_length, 0}, - {store_qos0, false}], - Q = ?Q:new("infinity_priority_queue", Opts, alarm_fun()), - 0 = ?Q:max_len(Q), - Qx = lists:foldl(fun(I, AccQ) -> - AccQ1 = - ?Q:in(#mqtt_message{topic = <<"t1">>, qos = 1, payload = iolist_to_binary([I])}, AccQ), - ?Q:in(#mqtt_message{topic = <<"t">>, qos = 1, payload = iolist_to_binary([I])}, AccQ1) - end, Q, lists:seq(1, 255)), - 510 = ?Q:len(Qx), - [{len, 510}, {max_len, 0}, {dropped, 0}] = ?Q:stats(Qx). - -t_priority_mqueue2(_) -> - Opts = [{type, priority}, - {max_length, 2}, - {low_watermark, 0.2}, - {high_watermark, 0.6}, - {store_qos0, false}], - Q = ?Q:new("priority_queue2_test", Opts, alarm_fun()), - 2 = ?Q:max_len(Q), - Q1 = ?Q:in(#mqtt_message{topic = <<"x">>, qos = 1, payload = <<1>>}, Q), - Q2 = ?Q:in(#mqtt_message{topic = <<"x">>, qos = 1, payload = <<2>>}, Q1), - Q3 = ?Q:in(#mqtt_message{topic = <<"y">>, qos = 1, payload = <<3>>}, Q2), - Q4 = ?Q:in(#mqtt_message{topic = <<"y">>, qos = 1, payload = <<4>>}, Q3), - 4 = ?Q:len(Q4), - {{value, _Val}, Q5} = ?Q:out(Q4), - 3 = ?Q:len(Q5). - -alarm_fun() -> fun(_, _) -> alarm_fun() end. - diff --git a/test/emqttd_protocol_SUITE.erl b/test/emqttd_protocol_SUITE.erl deleted file mode 100644 index 3401860e6..000000000 --- a/test/emqttd_protocol_SUITE.erl +++ /dev/null @@ -1,376 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_protocol_SUITE). - --compile(export_all). - --import(emqttd_serializer, [serialize/1]). - --include("emqttd.hrl"). - --include_lib("eunit/include/eunit.hrl"). - --include("emqttd_protocol.hrl"). - -all() -> - [{group, parser}, - {group, serializer}, - {group, packet}, - {group, message}]. - -groups() -> - [{parser, [], - [parse_connect, - parse_bridge, - parse_publish, - parse_puback, - parse_pubrec, - parse_pubrel, - parse_pubcomp, - parse_subscribe, - parse_unsubscribe, - parse_pingreq, - parse_disconnect]}, - {serializer, [], - [serialize_connect, - serialize_connack, - serialize_publish, - serialize_puback, - serialize_pubrel, - serialize_subscribe, - serialize_suback, - serialize_unsubscribe, - serialize_unsuback, - serialize_pingreq, - serialize_pingresp, - serialize_disconnect]}, - {packet, [], - [packet_proto_name, - packet_type_name, - packet_connack_name, - packet_format]}, - {message, [], - [message_make, - message_from_packet, - message_flag]}]. - -%%-------------------------------------------------------------------- -%% Parse Cases -%%-------------------------------------------------------------------- - -parse_connect(_) -> - Parser = emqttd_parser:initial_state(), - %% CONNECT(Q0, R0, D0, ClientId=mosqpub/10451-iMac.loca, ProtoName=MQIsdp, ProtoVsn=3, CleanSess=true, KeepAlive=60, Username=undefined, Password=undefined) - V31ConnBin = <<16,37,0,6,77,81,73,115,100,112,3,2,0,60,0,23,109,111,115,113,112,117,98,47,49,48,52,53,49,45,105,77,97,99,46,108,111,99,97>>, - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT, - dup = false, - qos = 0, - retain = false}, - variable = #mqtt_packet_connect{proto_ver = 3, - proto_name = <<"MQIsdp">>, - client_id = <<"mosqpub/10451-iMac.loca">>, - clean_sess = true, - keep_alive = 60}}, <<>>} = emqttd_parser:parse(V31ConnBin, Parser), - %% CONNECT(Q0, R0, D0, ClientId=mosqpub/10451-iMac.loca, ProtoName=MQTT, ProtoVsn=4, CleanSess=true, KeepAlive=60, Username=undefined, Password=undefined) - V311ConnBin = <<16,35,0,4,77,81,84,84,4,2,0,60,0,23,109,111,115,113,112,117,98,47,49,48,52,53,49,45,105,77,97,99,46,108,111,99,97>>, - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT, - dup = false, - qos = 0, - retain = false}, - variable = #mqtt_packet_connect{proto_ver = 4, - proto_name = <<"MQTT">>, - client_id = <<"mosqpub/10451-iMac.loca">>, - clean_sess = true, - keep_alive = 60 } }, <<>>} = emqttd_parser:parse(V311ConnBin, Parser), - - %% CONNECT(Qos=0, Retain=false, Dup=false, ClientId="", ProtoName=MQTT, ProtoVsn=4, CleanSess=true, KeepAlive=60) - V311ConnWithoutClientId = <<16,12,0,4,77,81,84,84,4,2,0,60,0,0>>, - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT, - dup = false, - qos = 0, - retain = false}, - variable = #mqtt_packet_connect{proto_ver = 4, - proto_name = <<"MQTT">>, - client_id = <<>>, - clean_sess = true, - keep_alive = 60 } }, <<>>} = emqttd_parser:parse(V311ConnWithoutClientId, Parser), - %%CONNECT(Q0, R0, D0, ClientId=mosqpub/10452-iMac.loca, ProtoName=MQIsdp, ProtoVsn=3, CleanSess=true, KeepAlive=60, - %% Username=test, Password=******, Will(Qos=1, Retain=false, Topic=/will, Msg=willmsg)) - ConnBinWithWill = <<16,67,0,6,77,81,73,115,100,112,3,206,0,60,0,23,109,111,115,113,112,117,98,47,49,48,52,53,50,45,105,77,97,99,46,108,111,99,97,0,5,47,119,105,108,108,0,7,119,105,108,108,109,115,103,0,4,116,101,115,116,0,6,112,117,98,108,105,99>>, - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT, - dup = false, - qos = 0, - retain = false}, - variable = #mqtt_packet_connect{proto_ver = 3, - proto_name = <<"MQIsdp">>, - client_id = <<"mosqpub/10452-iMac.loca">>, - clean_sess = true, - keep_alive = 60, - will_retain = false, - will_qos = 1, - will_flag = true, - will_topic = <<"/will">>, - will_msg = <<"willmsg">>, - username = <<"test">>, - password = <<"public">>}}, <<>>} = emqttd_parser:parse(ConnBinWithWill, Parser), - ok. - -parse_bridge(_) -> - Parser = emqttd_parser:initial_state(), - Data = <<16,86,0,6,77,81,73,115,100,112,131,44,0,60,0,19,67,95,48,48,58,48,67,58,50,57,58,50,66,58,55,55,58,53,50, - 0,48,36,83,89,83,47,98,114,111,107,101,114,47,99,111,110,110,101,99,116,105,111,110,47,67,95,48,48,58,48, - 67,58,50,57,58,50,66,58,55,55,58,53,50,47,115,116,97,116,101,0,1,48>>, - - %% CONNECT(Q0, R0, D0, ClientId=C_00:0C:29:2B:77:52, ProtoName=MQIsdp, ProtoVsn=131, CleanSess=false, KeepAlive=60, - %% Username=undefined, Password=undefined, Will(Q1, R1, Topic=$SYS/broker/connection/C_00:0C:29:2B:77:52/state, Msg=0)) - {ok, #mqtt_packet{variable = Variable}, <<>>} = emqttd_parser:parse(Data, Parser), - #mqtt_packet_connect{client_id = <<"C_00:0C:29:2B:77:52">>, - proto_ver = 16#03, - proto_name = <<"MQIsdp">>, - will_retain = true, - will_qos = 1, - will_flag = true, - clean_sess = false, - keep_alive = 60, - will_topic = <<"$SYS/broker/connection/C_00:0C:29:2B:77:52/state">>, - will_msg = <<"0">>} = Variable. - -parse_publish(_) -> - Parser = emqttd_parser:initial_state(), - %%PUBLISH(Qos=1, Retain=false, Dup=false, TopicName=a/b/c, PacketId=1, Payload=<<"hahah">>) - PubBin = <<50,14,0,5,97,47,98,47,99,0,1,104,97,104,97,104>>, - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, - dup = false, - qos = 1, - retain = false}, - variable = #mqtt_packet_publish{topic_name = <<"a/b/c">>, - packet_id = 1}, - payload = <<"hahah">> }, <<>>} = emqttd_parser:parse(PubBin, Parser), - - %PUBLISH(Qos=0, Retain=false, Dup=false, TopicName=xxx/yyy, PacketId=undefined, Payload=<<"hello">>) - %DISCONNECT(Qos=0, Retain=false, Dup=false) - PubBin1 = <<48,14,0,7,120,120,120,47,121,121,121,104,101,108,108,111,224,0>>, - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, - dup = false, - qos = 0, - retain = false}, - variable = #mqtt_packet_publish{topic_name = <<"xxx/yyy">>, - packet_id = undefined}, - payload = <<"hello">> }, <<224,0>>} = emqttd_parser:parse(PubBin1, Parser), - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?DISCONNECT, - dup = false, - qos = 0, - retain = false}}, <<>>} = emqttd_parser:parse(<<224, 0>>, Parser). - -parse_puback(_) -> - Parser = emqttd_parser:initial_state(), - %%PUBACK(Qos=0, Retain=false, Dup=false, PacketId=1) - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBACK, - dup = false, - qos = 0, - retain = false}}, <<>>} = emqttd_parser:parse(<<64,2,0,1>>, Parser). -parse_pubrec(_) -> - Parser = emqttd_parser:initial_state(), - %%PUBREC(Qos=0, Retain=false, Dup=false, PacketId=1) - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREC, - dup = false, - qos = 0, - retain = false}}, <<>>} = emqttd_parser:parse(<<5:4,0:4,2,0,1>>, Parser). - -parse_pubrel(_) -> - Parser = emqttd_parser:initial_state(), - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBREL, - dup = false, - qos = 1, - retain = false}}, <<>>} = emqttd_parser:parse(<<6:4,2:4,2,0,1>>, Parser). - -parse_pubcomp(_) -> - Parser = emqttd_parser:initial_state(), - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PUBCOMP, - dup = false, - qos = 0, - retain = false}}, <<>>} = emqttd_parser:parse(<<7:4,0:4,2,0,1>>, Parser). - -parse_subscribe(_) -> - Parser = emqttd_parser:initial_state(), - %% SUBSCRIBE(Q1, R0, D0, PacketId=2, TopicTable=[{<<"TopicA">>,2}]) - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?SUBSCRIBE, - dup = false, - qos = 1, - retain = false}, - variable = #mqtt_packet_subscribe{packet_id = 2, - topic_table = [{<<"TopicA">>,2}]} }, <<>>} - = emqttd_parser:parse(<<130,11,0,2,0,6,84,111,112,105,99,65,2>>, Parser). - -parse_unsubscribe(_) -> - Parser = emqttd_parser:initial_state(), - %% UNSUBSCRIBE(Q1, R0, D0, PacketId=2, TopicTable=[<<"TopicA">>]) - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?UNSUBSCRIBE, - dup = false, - qos = 1, - retain = false}, - variable = #mqtt_packet_unsubscribe{packet_id = 2, - topics = [<<"TopicA">>]}}, <<>>} - = emqttd_parser:parse(<<162,10,0,2,0,6,84,111,112,105,99,65>>, Parser). - -parse_pingreq(_) -> - Parser = emqttd_parser:initial_state(), - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?PINGREQ, - dup = false, - qos = 0, - retain = false}}, <<>>} - = emqttd_parser:parse(<>, Parser). - -parse_disconnect(_) -> - Parser = emqttd_parser:initial_state(), - %DISCONNECT(Qos=0, Retain=false, Dup=false) - Bin = <<224, 0>>, - {ok, #mqtt_packet{header = #mqtt_packet_header{type = ?DISCONNECT, - dup = false, - qos = 0, - retain = false}}, <<>>} = emqttd_parser:parse(Bin, Parser). - -%%-------------------------------------------------------------------- -%% Serialize Cases -%%-------------------------------------------------------------------- - -serialize_connect(_) -> - serialize(?CONNECT_PACKET(#mqtt_packet_connect{})), - serialize(?CONNECT_PACKET(#mqtt_packet_connect{ - client_id = <<"clientId">>, - will_qos = ?QOS1, - will_flag = true, - will_retain = true, - will_topic = <<"will">>, - will_msg = <<"haha">>, - clean_sess = true})). - -serialize_connack(_) -> - ConnAck = #mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}, - variable = #mqtt_packet_connack{ack_flags = 0, return_code = 0}}, - ?assertEqual(<<32,2,0,0>>, iolist_to_binary(serialize(ConnAck))). - -serialize_publish(_) -> - serialize(?PUBLISH_PACKET(?QOS_0, <<"Topic">>, undefined, <<"Payload">>)), - serialize(?PUBLISH_PACKET(?QOS_1, <<"Topic">>, 938, <<"Payload">>)), - serialize(?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 99, long_payload())). - -serialize_puback(_) -> - serialize(?PUBACK_PACKET(?PUBACK, 10384)). - -serialize_pubrel(_) -> - serialize(?PUBREL_PACKET(10384)). - -serialize_subscribe(_) -> - TopicTable = [{<<"TopicQos0">>, ?QOS_0}, {<<"TopicQos1">>, ?QOS_1}, {<<"TopicQos2">>, ?QOS_2}], - serialize(?SUBSCRIBE_PACKET(10, TopicTable)). - -serialize_suback(_) -> - serialize(?SUBACK_PACKET(10, [?QOS_0, ?QOS_1, 128])). - -serialize_unsubscribe(_) -> - serialize(?UNSUBSCRIBE_PACKET(10, [<<"Topic1">>, <<"Topic2">>])). - -serialize_unsuback(_) -> - serialize(?UNSUBACK_PACKET(10)). - -serialize_pingreq(_) -> - serialize(?PACKET(?PINGREQ)). - -serialize_pingresp(_) -> - serialize(?PACKET(?PINGRESP)). - -serialize_disconnect(_) -> - serialize(?PACKET(?DISCONNECT)). - -long_payload() -> - iolist_to_binary(["payload." || _I <- lists:seq(1, 100)]). - -%%-------------------------------------------------------------------- -%% Packet Cases -%%-------------------------------------------------------------------- - -packet_proto_name(_) -> - ?assertEqual(<<"MQIsdp">>, emqttd_packet:protocol_name(3)), - ?assertEqual(<<"MQTT">>, emqttd_packet:protocol_name(4)). - -packet_type_name(_) -> - ?assertEqual('CONNECT', emqttd_packet:type_name(?CONNECT)), - ?assertEqual('UNSUBSCRIBE', emqttd_packet:type_name(?UNSUBSCRIBE)). - -packet_connack_name(_) -> - ?assertEqual('CONNACK_ACCEPT', emqttd_packet:connack_name(?CONNACK_ACCEPT)), - ?assertEqual('CONNACK_PROTO_VER', emqttd_packet:connack_name(?CONNACK_PROTO_VER)), - ?assertEqual('CONNACK_INVALID_ID', emqttd_packet:connack_name(?CONNACK_INVALID_ID)), - ?assertEqual('CONNACK_SERVER', emqttd_packet:connack_name(?CONNACK_SERVER)), - ?assertEqual('CONNACK_CREDENTIALS', emqttd_packet:connack_name(?CONNACK_CREDENTIALS)), - ?assertEqual('CONNACK_AUTH', emqttd_packet:connack_name(?CONNACK_AUTH)). - -packet_format(_) -> - io:format("~s", [emqttd_packet:format(?CONNECT_PACKET(#mqtt_packet_connect{}))]), - io:format("~s", [emqttd_packet:format(?CONNACK_PACKET(?CONNACK_SERVER))]), - io:format("~s", [emqttd_packet:format(?PUBLISH_PACKET(?QOS_1, 1))]), - io:format("~s", [emqttd_packet:format(?PUBLISH_PACKET(?QOS_2, <<"topic">>, 10, <<"payload">>))]), - io:format("~s", [emqttd_packet:format(?PUBACK_PACKET(?PUBACK, 98))]), - io:format("~s", [emqttd_packet:format(?PUBREL_PACKET(99))]), - io:format("~s", [emqttd_packet:format(?SUBSCRIBE_PACKET(15, [{<<"topic">>, ?QOS0}, {<<"topic1">>, ?QOS1}]))]), - io:format("~s", [emqttd_packet:format(?SUBACK_PACKET(40, [?QOS0, ?QOS1]))]), - io:format("~s", [emqttd_packet:format(?UNSUBSCRIBE_PACKET(89, [<<"t">>, <<"t2">>]))]), - io:format("~s", [emqttd_packet:format(?UNSUBACK_PACKET(90))]). - -%%-------------------------------------------------------------------- -%% Message Cases -%%-------------------------------------------------------------------- - -message_make(_) -> - Msg = emqttd_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), - ?assertEqual(0, Msg#mqtt_message.qos), - Msg1 = emqttd_message:make(<<"clientid">>, qos2, <<"topic">>, <<"payload">>), - ?assert(is_binary(Msg1#mqtt_message.id)), - ?assertEqual(2, Msg1#mqtt_message.qos). - -message_from_packet(_) -> - Msg = emqttd_message:from_packet(?PUBLISH_PACKET(1, <<"topic">>, 10, <<"payload">>)), - ?assertEqual(1, Msg#mqtt_message.qos), - ?assertEqual(10, Msg#mqtt_message.pktid), - ?assertEqual(<<"topic">>, Msg#mqtt_message.topic), - WillMsg = emqttd_message:from_packet(#mqtt_packet_connect{will_flag = true, - will_topic = <<"WillTopic">>, - will_msg = <<"WillMsg">>}), - ?assertEqual(<<"WillTopic">>, WillMsg#mqtt_message.topic), - ?assertEqual(<<"WillMsg">>, WillMsg#mqtt_message.payload), - - Msg2 = emqttd_message:from_packet(<<"username">>, <<"clientid">>, - ?PUBLISH_PACKET(1, <<"topic">>, 20, <<"payload">>)), - ?assertEqual({<<"clientid">>, <<"username">>}, Msg2#mqtt_message.from), - io:format("~s", [emqttd_message:format(Msg2)]). - -message_flag(_) -> - Pkt = ?PUBLISH_PACKET(1, <<"t">>, 2, <<"payload">>), - Msg2 = emqttd_message:from_packet(<<"clientid">>, Pkt), - Msg3 = emqttd_message:set_flag(retain, Msg2), - Msg4 = emqttd_message:set_flag(dup, Msg3), - ?assert(Msg4#mqtt_message.dup), - ?assert(Msg4#mqtt_message.retain), - Msg5 = emqttd_message:set_flag(Msg4), - Msg6 = emqttd_message:unset_flag(dup, Msg5), - Msg7 = emqttd_message:unset_flag(retain, Msg6), - ?assertNot(Msg7#mqtt_message.dup), - ?assertNot(Msg7#mqtt_message.retain), - emqttd_message:unset_flag(Msg7), - emqttd_message:to_packet(Msg7). - diff --git a/test/emqttd_router_SUITE.erl b/test/emqttd_router_SUITE.erl deleted file mode 100644 index b305d699d..000000000 --- a/test/emqttd_router_SUITE.erl +++ /dev/null @@ -1,174 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_router_SUITE). - --compile(export_all). - --include("emqttd.hrl"). - --include_lib("eunit/include/eunit.hrl"). - --define(R, emqttd_router). - -all() -> - [{group, route}, - {group, local_route}]. - -groups() -> - [{route, [sequence], - [t_get_topics, - t_add_del_route, - t_match_route, - t_print, - t_has_route, - router_unused]}, - {local_route, [sequence], - [t_get_local_topics, - t_add_del_local_route, - t_match_local_route]}]. - -init_per_suite(Config) -> - ekka:start(), - ekka_mnesia:ensure_started(), - {ok, _R} = emqttd_router:start(), - Config. - -end_per_suite(_Config) -> - emqttd_router:stop(), - ekka:stop(), - ekka_mnesia:ensure_stopped(), - ekka_mnesia:delete_schema(). - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, _Config) -> - clear_tables(). - -t_get_topics(_) -> - ?R:add_route(<<"a/b/c">>), - ?R:add_route(<<"a/b/c">>), - ?R:add_route(<<"a/+/b">>), - ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), - ?R:del_route(<<"a/b/c">>), - ?R:del_route(<<"a/+/b">>), - ?assertEqual([], lists:sort(?R:topics())). - -t_add_del_route(_) -> - %%Node = node(), - ?R:add_route(<<"a/b/c">>), - ?R:add_route(<<"a/+/b">>), - ?R:del_route(<<"a/b/c">>), - ?R:del_route(<<"a/+/b">>). - -t_match_route(_) -> - Node = node(), - ?R:add_route(<<"a/b/c">>), - ?R:add_route(<<"a/+/c">>), - ?R:add_route(<<"a/b/#">>), - ?R:add_route(<<"#">>), - ?assertEqual([#mqtt_route{topic = <<"#">>, node = Node}, - #mqtt_route{topic = <<"a/+/c">>, node = Node}, - #mqtt_route{topic = <<"a/b/#">>, node = Node}, - #mqtt_route{topic = <<"a/b/c">>, node = Node}], - lists:sort(?R:match(<<"a/b/c">>))). - -t_has_route(_) -> - ?R:add_route(<<"devices/+/messages">>), - ?assert(?R:has_route(<<"devices/+/messages">>)). - -t_get_local_topics(_) -> - ?R:add_local_route(<<"a/b/c">>), - ?R:add_local_route(<<"x/+/y">>), - ?R:add_local_route(<<"z/#">>), - ?assertEqual([<<"z/#">>, <<"x/+/y">>, <<"a/b/c">>], ?R:local_topics()), - ?R:del_local_route(<<"x/+/y">>), - ?R:del_local_route(<<"z/#">>), - ?assertEqual([<<"a/b/c">>], ?R:local_topics()). - -t_add_del_local_route(_) -> - Node = node(), - ?R:add_local_route(<<"a/b/c">>), - ?R:add_local_route(<<"x/+/y">>), - ?R:add_local_route(<<"z/#">>), - ?assertEqual([{<<"a/b/c">>, Node}, - {<<"x/+/y">>, Node}, - {<<"z/#">>, Node}], - lists:sort(?R:get_local_routes())), - ?R:del_local_route(<<"x/+/y">>), - ?R:del_local_route(<<"z/#">>), - ?assertEqual([{<<"a/b/c">>, Node}], lists:sort(?R:get_local_routes())). - -t_match_local_route(_) -> - ?R:add_local_route(<<"$SYS/#">>), - ?R:add_local_route(<<"a/b/c">>), - ?R:add_local_route(<<"a/+/c">>), - ?R:add_local_route(<<"a/b/#">>), - ?R:add_local_route(<<"#">>), - Matched = [Topic || #mqtt_route{topic = {local, Topic}} <- ?R:match_local(<<"a/b/c">>)], - ?assertEqual([<<"#">>, <<"a/+/c">>, <<"a/b/#">>, <<"a/b/c">>], lists:sort(Matched)). - -clear_tables() -> - ?R:clean_local_routes(), - lists:foreach(fun mnesia:clear_table/1, [mqtt_route, mqtt_trie, mqtt_trie_node]). - -%%-------------------------------------------------------------------- -%% Router Test -%%-------------------------------------------------------------------- - -router_add_del(_) -> - %% Add - ?R:add_route(<<"#">>), - ?R:add_route(<<"a/b/c">>), - ?R:add_route(<<"+/#">>), - Routes = [R1, R2 | _] = [ - #mqtt_route{topic = <<"#">>, node = node()}, - #mqtt_route{topic = <<"+/#">>, node = node()}, - #mqtt_route{topic = <<"a/b/c">>, node = node()}], - Routes = lists:sort(?R:match(<<"a/b/c">>)), - - %% Batch Add - lists:foreach(fun(R) -> ?R:add_route(R) end, Routes), - Routes = lists:sort(?R:match(<<"a/b/c">>)), - - %% Del - ?R:del_route(<<"a/b/c">>), - [R1, R2] = lists:sort(?R:match(<<"a/b/c">>)), - {atomic, []} = mnesia:transaction(fun emqttd_trie:lookup/1, [<<"a/b/c">>]), - - %% Batch Del - R3 = #mqtt_route{topic = <<"#">>, node = 'a@127.0.0.1'}, - ?R:add_route(R3), - ?R:del_route(R1), - ?R:del_route(R2), - ?R:del_route(R3), - [] = lists:sort(?R:match(<<"a/b/c">>)). - -t_print(_) -> - Routes = [#mqtt_route{topic = <<"a/b/c">>, node = node()}, - #mqtt_route{topic = <<"#">>, node = node()}, - #mqtt_route{topic = <<"+/#">>, node = node()}], - lists:foreach(fun(R) -> ?R:add_route(R) end, Routes), - ?R:print(<<"a/b/c">>), - ?R:del_route(<<"+/#">>), - ?R:del_route(<<"a/b/c">>), - ?R:del_route(<<"#">>). - -router_unused(_) -> - gen_server:call(emqttd_router, bad_call), - gen_server:cast(emqttd_router, bad_msg), - emqttd_router ! bad_info. diff --git a/test/emqttd_trie_SUITE.erl b/test/emqttd_trie_SUITE.erl deleted file mode 100644 index 629531934..000000000 --- a/test/emqttd_trie_SUITE.erl +++ /dev/null @@ -1,135 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_trie_SUITE). - --compile(export_all). - --include("emqttd_trie.hrl"). - --define(TRIE, emqttd_trie). - --include_lib("eunit/include/eunit.hrl"). - -all() -> - [t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. - -init_per_suite(Config) -> - ekka_mnesia:ensure_started(), - ?TRIE:mnesia(boot), - ?TRIE:mnesia(copy), - Config. - -end_per_suite(_Config) -> - ekka_mnesia:ensure_stopped(), - ekka_mnesia:delete_schema(). - -init_per_testcase(_TestCase, Config) -> - Config. - -end_per_testcase(_TestCase, _Config) -> - clear_tables(). - -t_insert(_) -> - TN = #trie_node{node_id = <<"sensor">>, - edge_count = 3, - topic = <<"sensor">>, - flags = undefined}, - {atomic, [TN]} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor">>), - ?TRIE:lookup(<<"sensor">>) - end). - -t_match(_) -> - Machted = [<<"sensor/+/#">>, <<"sensor/#">>], - {atomic, Machted} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/+/#">>), - ?TRIE:insert(<<"sensor/#">>), - ?TRIE:match(<<"sensor/1">>) - end). - -t_match2(_) -> - Matched = {[<<"+/+/#">>, <<"+/#">>, <<"#">>], []}, - {atomic, Matched} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"#">>), - ?TRIE:insert(<<"+/#">>), - ?TRIE:insert(<<"+/+/#">>), - {?TRIE:match(<<"a/b/c">>), - ?TRIE:match(<<"$SYS/broker/zenmq">>)} - end). - -t_match3(_) -> - Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], - mnesia:transaction(fun() -> [emqttd_trie:insert(Topic) || Topic <- Topics] end), - Matched = mnesia:async_dirty(fun emqttd_trie:match/1, [<<"a/b/c">>]), - ?assertEqual(4, length(Matched)), - SysMatched = mnesia:async_dirty(fun emqttd_trie:match/1, [<<"$SYS/a/b/c">>]), - ?assertEqual([<<"$SYS/#">>], SysMatched). - -t_delete(_) -> - TN = #trie_node{node_id = <<"sensor/1">>, - edge_count = 2, - topic = undefined, - flags = undefined}, - {atomic, [TN]} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/1/#">>), - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), - ?TRIE:delete(<<"sensor/1/metric/2">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:delete(<<"sensor/1/metric">>), - ?TRIE:lookup(<<"sensor/1">>) - end). - -t_delete2(_) -> - {atomic, {[], []}} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor">>), - ?TRIE:insert(<<"sensor/1/metric/2">>), - ?TRIE:insert(<<"sensor/1/metric/3">>), - ?TRIE:delete(<<"sensor">>), - ?TRIE:delete(<<"sensor/1/metric/2">>), - ?TRIE:delete(<<"sensor/1/metric/3">>), - {?TRIE:lookup(<<"sensor">>), - ?TRIE:lookup(<<"sensor/1">>)} - end). - -t_delete3(_) -> - {atomic, {[], []}} = mnesia:transaction( - fun() -> - ?TRIE:insert(<<"sensor/+">>), - ?TRIE:insert(<<"sensor/+/metric/2">>), - ?TRIE:insert(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor/+/metric/2">>), - ?TRIE:delete(<<"sensor/+/metric/3">>), - ?TRIE:delete(<<"sensor">>), - ?TRIE:delete(<<"sensor/+">>), - ?TRIE:delete(<<"sensor/+/unknown">>), - {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} - end). - -clear_tables() -> - lists:foreach(fun mnesia:clear_table/1, [mqtt_trie, mqtt_trie_node]). - diff --git a/test/emqx_SUITE.erl b/test/emqx_SUITE.erl new file mode 100644 index 000000000..c6bea732a --- /dev/null +++ b/test/emqx_SUITE.erl @@ -0,0 +1,210 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(APP, emqx). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). + +-include("emqx_mqtt.hrl"). + +-record(ssl_socket, {tcp, ssl}). + + +-define(CLIENT, ?CONNECT_PACKET(#mqtt_packet_connect{ + client_id = <<"mqtt_client">>, + username = <<"admin">>, + password = <<"public">>})). + +-define(CLIENT2, ?CONNECT_PACKET(#mqtt_packet_connect{ + username = <<"admin">>, + clean_start = false, + password = <<"public">>})). + +-define(CLIENT3, ?CONNECT_PACKET(#mqtt_packet_connect{ + username = <<"admin">>, + proto_ver = ?MQTT_PROTO_V5, + clean_start = false, + password = <<"public">>, + will_props = #{'Will-Delay-Interval' => 2}})). + +-define(SUBCODE, [0]). + +-define(PACKETID, 1). + +-define(PUBQOS, 1). + +-define(SUBPACKET, ?SUBSCRIBE_PACKET(?PACKETID, [{<<"sub/topic">>, ?DEFAULT_SUBOPTS}])). + +-define(PUBPACKET, ?PUBLISH_PACKET(?PUBQOS, <<"sub/topic">>, ?PACKETID, <<"publish">>)). + +-define(PAYLOAD, [{type,"dsmSimulationData"}, + {id, 9999}, + {status, "running"}, + {soc, 1536702170}, + {fracsec, 451000}, + {data, lists:seq(1, 20480)}]). + +-define(BIG_PUBPACKET, ?PUBLISH_PACKET(?PUBQOS, <<"sub/topic">>, ?PACKETID, emqx_json:encode(?PAYLOAD))). + +all() -> + [{group, connect}, + {group, publish}]. + +groups() -> + [{connect, [non_parallel_tests], + [ + mqtt_connect, + mqtt_connect_with_tcp, + mqtt_connect_with_will_props, + mqtt_connect_with_ssl_oneway, + mqtt_connect_with_ssl_twoway, + mqtt_connect_with_ws + ]}, + {publish, [non_parallel_tests], + [ + packet_size + ]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +%%-------------------------------------------------------------------- +%% Protocol Test +%%-------------------------------------------------------------------- +mqtt_connect(_) -> + %% Issue #599 + %% Empty clientId and clean_session = false + ?assertEqual(<<32,2,0,2>>, connect_broker_(<<16,12,0,4,77,81,84,84,4,0,0,90,0,0>>, 4)), + %% Empty clientId and clean_session = true + ?assertEqual(<<32,2,0,0>>, connect_broker_(<<16,12,0,4,77,81,84,84,4,2,0,90,0,0>>, 4)). + +connect_broker_(Packet, RecvSize) -> + {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), + emqx_client_sock:send(Sock, Packet), + {ok, Data} = gen_tcp:recv(Sock, RecvSize, 3000), + emqx_client_sock:close(Sock), + Data. + +mqtt_connect_with_tcp(_) -> + %% Issue #599 + %% Empty clientId and clean_session = false + {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), + Packet = raw_send_serialize(?CLIENT2), + emqx_client_sock:send(Sock, Packet), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?CONNACK_INVALID_ID), _} = raw_recv_pase(Data), + emqx_client_sock:close(Sock). + +mqtt_connect_with_will_props(_) -> + %% Issue #599 + %% Empty clientId and clean_session = false + {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), + Packet = raw_send_serialize(?CLIENT3), + emqx_client_sock:send(Sock, Packet), + emqx_client_sock:close(Sock). + +mqtt_connect_with_ssl_oneway(_) -> + emqx:shutdown(), + emqx_ct_broker_helpers:change_opts(ssl_oneway), + emqx:start(), + ClientSsl = emqx_ct_broker_helpers:client_ssl(), + {ok, #ssl_socket{tcp = _Sock1, ssl = SslSock} = Sock} + = emqx_client_sock:connect("127.0.0.1", 8883, [{ssl_opts, ClientSsl}], 3000), + Packet = raw_send_serialize(?CLIENT), + emqx_client_sock:setopts(Sock, [{active, once}]), + emqx_client_sock:send(Sock, Packet), + ?assert( + receive {ssl, _, ConAck}-> + {ok, ?CONNACK_PACKET(?CONNACK_ACCEPT), _} = raw_recv_pase(ConAck), true + after 1000 -> + false + end), + ssl:close(SslSock). + +mqtt_connect_with_ssl_twoway(_Config) -> + emqx:shutdown(), + emqx_ct_broker_helpers:change_opts(ssl_twoway), + emqx:start(), + ClientSsl = emqx_ct_broker_helpers:client_ssl_twoway(), + {ok, #ssl_socket{tcp = _Sock1, ssl = SslSock} = Sock} + = emqx_client_sock:connect("127.0.0.1", 8883, [{ssl_opts, ClientSsl}], 3000), + Packet = raw_send_serialize(?CLIENT), + emqx_client_sock:setopts(Sock, [{active, once}]), + emqx_client_sock:send(Sock, Packet), + timer:sleep(500), + ?assert( + receive {ssl, _, Data}-> + {ok, ?CONNACK_PACKET(?CONNACK_ACCEPT), _} = raw_recv_pase(Data), true + after 1000 -> + false + end), + ssl:close(SslSock), + emqx_client_sock:close(Sock). + +mqtt_connect_with_ws(_Config) -> + WS = rfc6455_client:new("ws://127.0.0.1:8083" ++ "/mqtt", self()), + {ok, _} = rfc6455_client:open(WS), + + %% Connect Packet + Packet = raw_send_serialize(?CLIENT), + ok = rfc6455_client:send_binary(WS, Packet), + {binary, CONACK} = rfc6455_client:recv(WS), + {ok, ?CONNACK_PACKET(?CONNACK_ACCEPT), _} = raw_recv_pase(CONACK), + + %% Sub Packet + SubPacket = raw_send_serialize(?SUBPACKET), + rfc6455_client:send_binary(WS, SubPacket), + {binary, SubAck} = rfc6455_client:recv(WS), + {ok, ?SUBACK_PACKET(?PACKETID, ?SUBCODE), _} = raw_recv_pase(SubAck), + + %% Pub Packet QoS 1 + PubPacket = raw_send_serialize(?PUBPACKET), + rfc6455_client:send_binary(WS, PubPacket), + {binary, PubAck} = rfc6455_client:recv(WS), + {ok, ?PUBACK_PACKET(?PACKETID), _} = raw_recv_pase(PubAck), + {close, _} = rfc6455_client:close(WS), + ok. + +%%issue 1811 +packet_size(_Config) -> + {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), + Packet = raw_send_serialize(?CLIENT), + emqx_client_sock:send(Sock, Packet), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?CONNACK_ACCEPT), _} = raw_recv_pase(Data), + + %% Pub Packet QoS 1 + PubPacket = raw_send_serialize(?BIG_PUBPACKET), + emqx_client_sock:send(Sock, PubPacket), + {ok, Data1} = gen_tcp:recv(Sock, 0), + {ok, ?PUBACK_PACKET(?PACKETID), _} = raw_recv_pase(Data1), + emqx_client_sock:close(Sock). + +raw_send_serialize(Packet) -> + emqx_frame:serialize(Packet). + +raw_recv_pase(P) -> + emqx_frame:parse(P, {none, #{max_packet_size => ?MAX_PACKET_SIZE, + version => ?MQTT_PROTO_V4} }). diff --git a/test/emqx_SUITE_data/acl.conf b/test/emqx_SUITE_data/acl.conf new file mode 100644 index 000000000..3cb3b8c52 --- /dev/null +++ b/test/emqx_SUITE_data/acl.conf @@ -0,0 +1,29 @@ +%%-------------------------------------------------------------------- +%% +%% [ACL](https://github.com/emqtt/emqttd/wiki/ACL) +%% +%% -type who() :: all | binary() | +%% {ipaddr, esockd_access:cidr()} | +%% {client, binary()} | +%% {user, binary()}. +%% +%% -type access() :: subscribe | publish | pubsub. +%% +%% -type topic() :: binary(). +%% +%% -type rule() :: {allow, all} | +%% {allow, who(), access(), list(topic())} | +%% {deny, all} | +%% {deny, who(), access(), list(topic())}. +%% +%%-------------------------------------------------------------------- + +{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. + +{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. + +{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. + +{allow, all}. + + diff --git a/doc/.placeholder b/test/emqx_SUITE_data/loaded_plugins similarity index 100% rename from doc/.placeholder rename to test/emqx_SUITE_data/loaded_plugins diff --git a/test/emqx_SUITE_data/slave.config b/test/emqx_SUITE_data/slave.config new file mode 100644 index 000000000..1cdf851b5 --- /dev/null +++ b/test/emqx_SUITE_data/slave.config @@ -0,0 +1,45 @@ +[{emqx, + [{plugins_loaded_file,"loaded_plugins"}, + {plugins_etc_dir,"plugins/"}, + {broker_sys_interval,60}, + {cache_acl,true}, + {allow_anonymous,true}, + {license_file,"../../etc/emqx.lic"}, + {protocol,[{max_clientid_len,1024},{max_packet_size,65536}]}, + {client, + [{max_publish_rate,5},{idle_timeout,30000},{enable_stats,60000}]}, + {session, + [{max_subscriptions,0}, + {upgrade_qos,false}, + {max_inflight,32}, + {retry_interval,20000}, + {max_awaiting_rel,100}, + {await_rel_timeout,20000}, + {enable_stats,60000}, + {expiry_interval,7200000}]}, + {mqueue, + [{priority,[]}, + {type,simple}, + {max_length,infinity}, + {low_watermark,0.2}, + {high_watermark,0.6}, + {store_qos0,true}]}, + {pubsub,[{pool_size,8},{by_clientid,true},{async,true}]}, + {bridge,[{max_queue_len,10000},{ping_down_interval,1}]}, + {listeners, []}, + {sysmon, + [{long_gc,false}, + {long_schedule,240}, + {large_heap,8388608}, + {busy_port,false}, + {busy_dist_port,true}]}]}, + {gen_rpc, + [{socket_keepalive_count,2}, + {socket_keepalive_interval,5}, + {socket_keepalive_idle,5}, + {call_receive_timeout,15000}, + {authentication_timeout,5000}, + {send_timeout,5000}, + {connect_timeout,5000}, + {tcp_client_port,5369}, + {tcp_server_port,7369}]}]. diff --git a/test/emqx_access_SUITE.erl b/test/emqx_access_SUITE.erl new file mode 100644 index 000000000..b49c90d80 --- /dev/null +++ b/test/emqx_access_SUITE.erl @@ -0,0 +1,379 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_access_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(AC, emqx_access_control). +-define(CACHE, emqx_acl_cache). + +-import(emqx_access_rule, [compile/1, match/3]). + +all() -> + [{group, access_control}, + {group, acl_cache}, + {group, access_control_cache_mode}, + {group, access_rule} + ]. + +groups() -> + [{access_control, [sequence], + [reload_acl, + register_mod, + unregister_mod, + check_acl_1, + check_acl_2 + ]}, + {access_control_cache_mode, [], + [ + acl_cache_basic, + acl_cache_expiry, + acl_cache_cleanup, + acl_cache_full + ]}, + {acl_cache, [], [ + put_get_del_cache, + cache_update, + cache_expiry, + cache_replacement, + cache_cleanup, + cache_auto_emtpy, + cache_auto_cleanup + ]}, + {access_rule, [], + [compile_rule, + match_rule]}]. + +init_per_group(Group, Config) when Group =:= access_control; + Group =:= access_control_cache_mode -> + prepare_config(Group), + application:load(emqx), + Config; +init_per_group(_Group, Config) -> + Config. + +prepare_config(Group = access_control) -> + set_acl_config_file(Group), + application:set_env(emqx, enable_acl_cache, false); +prepare_config(Group = access_control_cache_mode) -> + set_acl_config_file(Group), + application:set_env(emqx, enable_acl_cache, true), + application:set_env(emqx, acl_cache_max_size, 100). + +set_acl_config_file(_Group) -> + Rules = [{allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]}, + {allow, {user, "testuser"}, subscribe, ["a/b/c", "d/e/f/#"]}, + {allow, {user, "admin"}, pubsub, ["a/b/c", "d/e/f/#"]}, + {allow, {client, "testClient"}, subscribe, ["testTopics/testClient"]}, + {allow, all, subscribe, ["clients/%c"]}, + {allow, all, pubsub, ["users/%u/#"]}, + {deny, all, subscribe, ["$SYS/#", "#"]}, + {deny, all}], + write_config("access_SUITE_acl.conf", Rules), + application:set_env(emqx, acl_file, "access_SUITE_acl.conf"). + + +write_config(Filename, Terms) -> + file:write_file(Filename, [io_lib:format("~tp.~n", [Term]) || Term <- Terms]). + +end_per_group(_Group, Config) -> + Config. + +init_per_testcase(_TestCase, Config) -> + %% {ok, _Pid} = + ?AC:start_link(), + Config. +end_per_testcase(_TestCase, _Config) -> + ok. + +per_testcase_config(acl_cache_full, Config) -> + Config; +per_testcase_config(_TestCase, Config) -> + Config. + + +%%-------------------------------------------------------------------- +%% emqx_access_control +%%-------------------------------------------------------------------- + +reload_acl(_) -> + [ok] = ?AC:reload_acl(). + +register_mod(_) -> + ok = ?AC:register_mod(acl, emqx_acl_test_mod, []), + {emqx_acl_test_mod, _, 0} = hd(?AC:lookup_mods(acl)), + ok = ?AC:register_mod(auth, emqx_auth_anonymous_test_mod,[]), + ok = ?AC:register_mod(auth, emqx_auth_dashboard, [], 99), + [{emqx_auth_dashboard, _, 99}, + {emqx_auth_anonymous_test_mod, _, 0}] = ?AC:lookup_mods(auth). + +unregister_mod(_) -> + ok = ?AC:register_mod(acl, emqx_acl_test_mod, []), + {emqx_acl_test_mod, _, 0} = hd(?AC:lookup_mods(acl)), + ok = ?AC:unregister_mod(acl, emqx_acl_test_mod), + timer:sleep(5), + {emqx_acl_internal, _, 0}= hd(?AC:lookup_mods(acl)), + ok = ?AC:register_mod(auth, emqx_auth_anonymous_test_mod,[]), + [{emqx_auth_anonymous_test_mod, _, 0}] = ?AC:lookup_mods(auth), + + ok = ?AC:unregister_mod(auth, emqx_auth_anonymous_test_mod), + timer:sleep(5), + [] = ?AC:lookup_mods(auth). + +check_acl_1(_) -> + SelfUser = #{client_id => <<"client1">>, username => <<"testuser">>}, + allow = ?AC:check_acl(SelfUser, subscribe, <<"users/testuser/1">>), + allow = ?AC:check_acl(SelfUser, subscribe, <<"clients/client1">>), + deny = ?AC:check_acl(SelfUser, subscribe, <<"clients/client1/x/y">>), + allow = ?AC:check_acl(SelfUser, publish, <<"users/testuser/1">>), + allow = ?AC:check_acl(SelfUser, subscribe, <<"a/b/c">>). +check_acl_2(_) -> + SelfUser = #{client_id => <<"client2">>, username => <<"xyz">>}, + deny = ?AC:check_acl(SelfUser, subscribe, <<"a/b/c">>). + +acl_cache_basic(_) -> + SelfUser = #{client_id => <<"client1">>, username => <<"testuser">>}, + not_found = ?CACHE:get_acl_cache(subscribe, <<"users/testuser/1">>), + not_found = ?CACHE:get_acl_cache(subscribe, <<"clients/client1">>), + + allow = ?AC:check_acl(SelfUser, subscribe, <<"users/testuser/1">>), + allow = ?AC:check_acl(SelfUser, subscribe, <<"clients/client1">>), + + allow = ?CACHE:get_acl_cache(subscribe, <<"users/testuser/1">>), + allow = ?CACHE:get_acl_cache(subscribe, <<"clients/client1">>), + ok. + +acl_cache_expiry(_) -> + application:set_env(emqx, acl_cache_ttl, 100), + SelfUser = #{client_id => <<"client1">>, username => <<"testuser">>}, + allow = ?AC:check_acl(SelfUser, subscribe, <<"clients/client1">>), + allow = ?CACHE:get_acl_cache(subscribe, <<"clients/client1">>), + ct:sleep(150), + not_found = ?CACHE:get_acl_cache(subscribe, <<"clients/client1">>), + ok. + +acl_cache_full(_) -> + application:set_env(emqx, acl_cache_max_size, 1), + + SelfUser = #{client_id => <<"client1">>, username => <<"testuser">>}, + allow = ?AC:check_acl(SelfUser, subscribe, <<"users/testuser/1">>), + allow = ?AC:check_acl(SelfUser, subscribe, <<"clients/client1">>), + + %% the older ones (the <<"users/testuser/1">>) will be evicted first + not_found = ?CACHE:get_acl_cache(subscribe, <<"users/testuser/1">>), + allow = ?CACHE:get_acl_cache(subscribe, <<"clients/client1">>), + ok. + +acl_cache_cleanup(_) -> + %% The acl cache will try to evict memory, if the size is full and the newest + %% cache entry is expired + application:set_env(emqx, acl_cache_ttl, 100), + application:set_env(emqx, acl_cache_max_size, 2), + + SelfUser = #{client_id => <<"client1">>, username => <<"testuser">>}, + allow = ?AC:check_acl(SelfUser, subscribe, <<"users/testuser/1">>), + allow = ?AC:check_acl(SelfUser, subscribe, <<"clients/client1">>), + + allow = ?CACHE:get_acl_cache(subscribe, <<"users/testuser/1">>), + allow = ?CACHE:get_acl_cache(subscribe, <<"clients/client1">>), + + ct:sleep(150), + %% now the cache is full and the newest one - "clients/client1" + %% should be expired, so we'll empty the cache before putting + %% the next cache entry + deny = ?AC:check_acl(SelfUser, subscribe, <<"#">>), + + not_found = ?CACHE:get_acl_cache(subscribe, <<"users/testuser/1">>), + not_found = ?CACHE:get_acl_cache(subscribe, <<"clients/client1">>), + deny = ?CACHE:get_acl_cache(subscribe, <<"#">>), + ok. + +put_get_del_cache(_) -> + application:set_env(emqx, acl_cache_ttl, 300000), + application:set_env(emqx, acl_cache_max_size, 30), + + not_found = ?CACHE:get_acl_cache(publish, <<"a">>), + ok = ?CACHE:put_acl_cache(publish, <<"a">>, allow), + allow = ?CACHE:get_acl_cache(publish, <<"a">>), + + not_found = ?CACHE:get_acl_cache(subscribe, <<"b">>), + ok = ?CACHE:put_acl_cache(subscribe, <<"b">>, deny), + deny = ?CACHE:get_acl_cache(subscribe, <<"b">>), + + 2 = ?CACHE:get_cache_size(), + ?assertEqual(?CACHE:cache_k(subscribe, <<"b">>), ?CACHE:get_newest_key()). + +cache_expiry(_) -> + application:set_env(emqx, acl_cache_ttl, 100), + application:set_env(emqx, acl_cache_max_size, 30), + ok = ?CACHE:put_acl_cache(subscribe, <<"a">>, allow), + allow = ?CACHE:get_acl_cache(subscribe, <<"a">>), + + ct:sleep(150), + not_found = ?CACHE:get_acl_cache(subscribe, <<"a">>), + + ok = ?CACHE:put_acl_cache(subscribe, <<"a">>, deny), + deny = ?CACHE:get_acl_cache(subscribe, <<"a">>), + + ct:sleep(150), + not_found = ?CACHE:get_acl_cache(subscribe, <<"a">>). + +cache_update(_) -> + application:set_env(emqx, acl_cache_ttl, 300000), + application:set_env(emqx, acl_cache_max_size, 30), + [] = ?CACHE:dump_acl_cache(), + + ok = ?CACHE:put_acl_cache(subscribe, <<"a">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"b">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"c">>, allow), + 3 = ?CACHE:get_cache_size(), + ?assertEqual(?CACHE:cache_k(publish, <<"c">>), ?CACHE:get_newest_key()), + + %% update the 2nd one + ok = ?CACHE:put_acl_cache(publish, <<"b">>, allow), + ct:pal("dump acl cache: ~p~n", [?CACHE:dump_acl_cache()]), + + 3 = ?CACHE:get_cache_size(), + ?assertEqual(?CACHE:cache_k(publish, <<"b">>), ?CACHE:get_newest_key()), + ?assertEqual(?CACHE:cache_k(subscribe, <<"a">>), ?CACHE:get_oldest_key()). + +cache_replacement(_) -> + application:set_env(emqx, acl_cache_ttl, 300000), + application:set_env(emqx, acl_cache_max_size, 3), + ok = ?CACHE:put_acl_cache(subscribe, <<"a">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"b">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"c">>, allow), + allow = ?CACHE:get_acl_cache(subscribe, <<"a">>), + allow = ?CACHE:get_acl_cache(publish, <<"b">>), + allow = ?CACHE:get_acl_cache(publish, <<"c">>), + 3 = ?CACHE:get_cache_size(), + ?assertEqual(?CACHE:cache_k(publish, <<"c">>), ?CACHE:get_newest_key()), + + ok = ?CACHE:put_acl_cache(publish, <<"d">>, deny), + 3 = ?CACHE:get_cache_size(), + ?assertEqual(?CACHE:cache_k(publish, <<"d">>), ?CACHE:get_newest_key()), + ?assertEqual(?CACHE:cache_k(publish, <<"b">>), ?CACHE:get_oldest_key()), + + ok = ?CACHE:put_acl_cache(publish, <<"e">>, deny), + 3 = ?CACHE:get_cache_size(), + ?assertEqual(?CACHE:cache_k(publish, <<"e">>), ?CACHE:get_newest_key()), + ?assertEqual(?CACHE:cache_k(publish, <<"c">>), ?CACHE:get_oldest_key()), + + not_found = ?CACHE:get_acl_cache(subscribe, <<"a">>), + not_found = ?CACHE:get_acl_cache(publish, <<"b">>), + allow = ?CACHE:get_acl_cache(publish, <<"c">>). + +cache_cleanup(_) -> + application:set_env(emqx, acl_cache_ttl, 100), + application:set_env(emqx, acl_cache_max_size, 30), + ok = ?CACHE:put_acl_cache(subscribe, <<"a">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"b">>, allow), + ct:sleep(150), + ok = ?CACHE:put_acl_cache(publish, <<"c">>, allow), + 3 = ?CACHE:get_cache_size(), + + ?CACHE:cleanup_acl_cache(), + ?assertEqual(?CACHE:cache_k(publish, <<"c">>), ?CACHE:get_oldest_key()), + 1 = ?CACHE:get_cache_size(). + +cache_auto_emtpy(_) -> + %% verify cache is emptied when cache full and even the newest + %% one is expired. + application:set_env(emqx, acl_cache_ttl, 100), + application:set_env(emqx, acl_cache_max_size, 3), + ok = ?CACHE:put_acl_cache(subscribe, <<"a">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"b">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"c">>, allow), + 3 = ?CACHE:get_cache_size(), + + ct:sleep(150), + ok = ?CACHE:put_acl_cache(subscribe, <<"d">>, deny), + 1 = ?CACHE:get_cache_size(). + +cache_auto_cleanup(_) -> + %% verify we'll cleanup expired entries when we got a exipired acl + %% from cache. + application:set_env(emqx, acl_cache_ttl, 100), + application:set_env(emqx, acl_cache_max_size, 30), + ok = ?CACHE:put_acl_cache(subscribe, <<"a">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"b">>, allow), + ct:sleep(150), + ok = ?CACHE:put_acl_cache(publish, <<"c">>, allow), + ok = ?CACHE:put_acl_cache(publish, <<"d">>, deny), + 4 = ?CACHE:get_cache_size(), + + %% "a" and "b" expires, while "c" and "d" not + not_found = ?CACHE:get_acl_cache(publish, <<"b">>), + 2 = ?CACHE:get_cache_size(), + + ct:sleep(150), %% now "c" and "d" expires + not_found = ?CACHE:get_acl_cache(publish, <<"c">>), + 0 = ?CACHE:get_cache_size(). + +%%-------------------------------------------------------------------- +%% emqx_access_rule +%%-------------------------------------------------------------------- + +compile_rule(_) -> + {allow, {'and', [{ipaddr, {{127,0,0,1}, {127,0,0,1}, 32}}, + {user, <<"user">>}]}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]} = + compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"user">>}]}, subscribe, ["$SYS/#", "#"]}), + {allow, {'or', [{ipaddr, {{127,0,0,1}, {127,0,0,1}, 32}}, + {user, <<"user">>}]}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]} = + compile({allow, {'or', [{ipaddr, "127.0.0.1"}, {user, <<"user">>}]}, subscribe, ["$SYS/#", "#"]}), + + {allow, {ipaddr, {{127,0,0,1}, {127,0,0,1}, 32}}, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]} = + compile({allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]}), + {allow, {user, <<"testuser">>}, subscribe, [ [<<"a">>, <<"b">>, <<"c">>], [<<"d">>, <<"e">>, <<"f">>, '#'] ]} = + compile({allow, {user, "testuser"}, subscribe, ["a/b/c", "d/e/f/#"]}), + {allow, {user, <<"admin">>}, pubsub, [ [<<"d">>, <<"e">>, <<"f">>, '#'] ]} = + compile({allow, {user, "admin"}, pubsub, ["d/e/f/#"]}), + {allow, {client, <<"testClient">>}, publish, [ [<<"testTopics">>, <<"testClient">>] ]} = + compile({allow, {client, "testClient"}, publish, ["testTopics/testClient"]}), + {allow, all, pubsub, [{pattern, [<<"clients">>, <<"%c">>]}]} = + compile({allow, all, pubsub, ["clients/%c"]}), + {allow, all, subscribe, [{pattern, [<<"users">>, <<"%u">>, '#']}]} = + compile({allow, all, subscribe, ["users/%u/#"]}), + {deny, all, subscribe, [ [<<"$SYS">>, '#'], ['#'] ]} = + compile({deny, all, subscribe, ["$SYS/#", "#"]}), + {allow, all} = compile({allow, all}), + {deny, all} = compile({deny, all}). + +match_rule(_) -> + User = #{client_id => <<"testClient">>, username => <<"TestUser">>, peername => {{127,0,0,1}, 2948}}, + User2 = #{client_id => <<"testClient">>, username => <<"TestUser">>, peername => {{192,168,0,10}, 3028}}, + + {matched, allow} = match(User, <<"Test/Topic">>, {allow, all}), + {matched, deny} = match(User, <<"Test/Topic">>, {deny, all}), + {matched, allow} = match(User, <<"Test/Topic">>, compile({allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]})), + {matched, allow} = match(User2, <<"Test/Topic">>, compile({allow, {ipaddr, "192.168.0.1/24"}, subscribe, ["$SYS/#", "#"]})), + {matched, allow} = match(User, <<"d/e/f/x">>, compile({allow, {user, "TestUser"}, subscribe, ["a/b/c", "d/e/f/#"]})), + nomatch = match(User, <<"d/e/f/x">>, compile({allow, {user, "admin"}, pubsub, ["d/e/f/#"]})), + {matched, allow} = match(User, <<"testTopics/testClient">>, compile({allow, {client, "testClient"}, publish, ["testTopics/testClient"]})), + {matched, allow} = match(User, <<"clients/testClient">>, compile({allow, all, pubsub, ["clients/%c"]})), + {matched, allow} = match(#{username => <<"user2">>}, <<"users/user2/abc/def">>, compile({allow, all, subscribe, ["users/%u/#"]})), + {matched, deny} = match(User, <<"d/e/f">>, compile({deny, all, subscribe, ["$SYS/#", "#"]})), + Rule = compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"WrongUser">>}]}, publish, <<"Topic">>}), + nomatch = match(User, <<"Topic">>, Rule), + AndRule = compile({allow, {'and', [{ipaddr, "127.0.0.1"}, {user, <<"TestUser">>}]}, publish, <<"Topic">>}), + {matched, allow} = match(User, <<"Topic">>, AndRule), + OrRule = compile({allow, {'or', [{ipaddr, "127.0.0.1"}, {user, <<"WrongUser">>}]}, publish, ["Topic"]}), + {matched, allow} = match(User, <<"Topic">>, OrRule). diff --git a/test/emqx_access_SUITE_data/acl.conf b/test/emqx_access_SUITE_data/acl.conf new file mode 100644 index 000000000..e5730b4c5 --- /dev/null +++ b/test/emqx_access_SUITE_data/acl.conf @@ -0,0 +1,15 @@ +{allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]}. + +{allow, {user, "testuser"}, subscribe, ["a/b/c", "d/e/f/#"]}. + +{allow, {user, "admin"}, pubsub, ["a/b/c", "d/e/f/#"]}. + +{allow, {client, "testClient"}, subscribe, ["testTopics/testClient"]}. + +{allow, all, subscribe, ["clients/%c"]}. + +{allow, all, pubsub, ["users/%u/#"]}. + +{deny, all, subscribe, ["$SYS/#", "#"]}. + +{deny, all}. diff --git a/test/emqx_access_SUITE_data/acl_deny_action.conf b/test/emqx_access_SUITE_data/acl_deny_action.conf new file mode 100644 index 000000000..753782605 --- /dev/null +++ b/test/emqx_access_SUITE_data/acl_deny_action.conf @@ -0,0 +1,4 @@ + +{deny, {user, "emqx"}, pubsub, ["acl_deny_action"]}. + +{allow, all}. diff --git a/test/emqttd_acl_test_mod.erl b/test/emqx_acl_test_mod.erl similarity index 77% rename from test/emqttd_acl_test_mod.erl rename to test/emqx_acl_test_mod.erl index 9ed34c263..131336cdd 100644 --- a/test/emqttd_acl_test_mod.erl +++ b/test/emqx_acl_test_mod.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,9 +11,8 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_acl_test_mod). +-module(emqx_acl_test_mod). %% ACL callbacks -export([init/1, check_acl/2, reload_acl/1, description/0]). @@ -30,3 +28,4 @@ reload_acl(_State) -> description() -> "Test ACL Mod". + diff --git a/test/emqttd_auth_anonymous_test_mod.erl b/test/emqx_auth_anonymous_test_mod.erl similarity index 71% rename from test/emqttd_auth_anonymous_test_mod.erl rename to test/emqx_auth_anonymous_test_mod.erl index 0f01be47f..e04841feb 100644 --- a/test/emqttd_auth_anonymous_test_mod.erl +++ b/test/emqx_auth_anonymous_test_mod.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,9 +11,8 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_auth_anonymous_test_mod). +-module(emqx_auth_anonymous_test_mod). %% ACL callbacks -export([init/1, check/3, description/0]). @@ -26,4 +24,4 @@ check(_Client, _Password, _Opts) -> allow. description() -> - "Test emqttd_auth_anonymous Mod". + "Test emqx_auth_anonymous Mod". diff --git a/test/emqttd_auth_dashboard.erl b/test/emqx_auth_dashboard.erl similarity index 72% rename from test/emqttd_auth_dashboard.erl rename to test/emqx_auth_dashboard.erl index 97ed17ea4..b8c742d3b 100644 --- a/test/emqttd_auth_dashboard.erl +++ b/test/emqx_auth_dashboard.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,9 +11,8 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_auth_dashboard). +-module(emqx_auth_dashboard). %% Auth callbacks -export([init/1, check/3, description/0]). @@ -26,4 +24,5 @@ check(_Client, _Password, _Opts) -> allow. description() -> - "Test emqttd_auth_dashboard Mod". + "Test Auth Mod". + diff --git a/test/emqx_banned_SUITE.erl b/test/emqx_banned_SUITE.erl new file mode 100644 index 000000000..7e7434e61 --- /dev/null +++ b/test/emqx_banned_SUITE.erl @@ -0,0 +1,53 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_banned_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> [t_banned_all]. + +t_banned_all(_) -> + emqx_ct_broker_helpers:run_setup_steps(), + emqx_banned:start_link(), + TimeNow = erlang:system_time(second), + Banned = #banned{who = {client_id, <<"TestClient">>}, + reason = <<"test">>, + by = <<"banned suite">>, + desc = <<"test">>, + until = TimeNow + 1}, + ok = emqx_banned:add(Banned), + % here is not expire banned test because its check interval is greater than 5 mins, but its effect has been confirmed + ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), + timer:sleep(2500), + ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), + ok = emqx_banned:add(Banned), + ?assert(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), + emqx_banned:delete({client_id, <<"TestClient">>}), + ?assertNot(emqx_banned:check(#{client_id => <<"TestClient">>, + username => undefined, + peername => {undefined, undefined}})), + emqx_ct_broker_helpers:run_teardown_steps(). + diff --git a/test/emqx_batch_SUITE.erl b/test/emqx_batch_SUITE.erl new file mode 100644 index 000000000..c4c69080b --- /dev/null +++ b/test/emqx_batch_SUITE.erl @@ -0,0 +1,50 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_batch_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [batch_full_commit, batch_linger_commit]. + +batch_full_commit(_) -> + B0 = emqx_batch:init(#{batch_size => 3, linger_ms => 2000, commit_fun => fun(_) -> ok end}), + B3 = lists:foldl(fun(E, B) -> emqx_batch:push(E, B) end, B0, [a, b, c]), + ?assertEqual(3, emqx_batch:size(B3)), + ?assertEqual([a, b, c], emqx_batch:items(B3)), + %% Trigger commit fun. + B4 = emqx_batch:push(a, B3), + ?assertEqual(0, emqx_batch:size(B4)), + ?assertEqual([], emqx_batch:items(B4)). + +batch_linger_commit(_) -> + CommitFun = fun(Q) -> ?assertEqual(3, length(Q)) end, + B0 = emqx_batch:init(#{batch_size => 3, linger_ms => 500, commit_fun => CommitFun}), + B3 = lists:foldl(fun(E, B) -> emqx_batch:push(E, B) end, B0, [a, b, c]), + ?assertEqual(3, emqx_batch:size(B3)), + ?assertEqual([a, b, c], emqx_batch:items(B3)), + receive + batch_linger_expired -> + B4 = emqx_batch:commit(B3), + ?assertEqual(0, emqx_batch:size(B4)), + ?assertEqual([], emqx_batch:items(B4)) + after + 1000 -> + error(linger_timer_not_triggered) + end. + diff --git a/test/emqx_bridge_SUITE.erl b/test/emqx_bridge_SUITE.erl new file mode 100644 index 000000000..f337e3b4e --- /dev/null +++ b/test/emqx_bridge_SUITE.erl @@ -0,0 +1,57 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_bridge_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + [bridge_test]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +bridge_test(_) -> + {ok, _Pid} = emqx_bridge:start_link(emqx, []), + #{msg := <<"start bridge successfully">>} + = emqx_bridge:start_bridge(emqx), + test_forwards(), + test_subscriptions(0), + test_subscriptions(1), + test_subscriptions(2), + #{msg := <<"stop bridge successfully">>} + = emqx_bridge:stop_bridge(emqx), + ok. + +test_forwards() -> + emqx_bridge:add_forward(emqx, <<"test_forwards">>), + [<<"test_forwards">>] = emqx_bridge:show_forwards(emqx), + emqx_bridge:del_forward(emqx, <<"test_forwards">>), + [] = emqx_bridge:show_forwards(emqx), + ok. + +test_subscriptions(QoS) -> + emqx_bridge:add_subscription(emqx, <<"test_subscriptions">>, QoS), + [{<<"test_subscriptions">>, QoS}] = emqx_bridge:show_subscriptions(emqx), + emqx_bridge:del_subscription(emqx, <<"test_subscriptions">>), + [] = emqx_bridge:show_subscriptions(emqx), + ok. diff --git a/test/emqx_broker_SUITE.erl b/test/emqx_broker_SUITE.erl new file mode 100644 index 000000000..b2031a61c --- /dev/null +++ b/test/emqx_broker_SUITE.erl @@ -0,0 +1,182 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_broker_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(APP, emqx). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +all() -> + [{group, pubsub}, + {group, session}, + {group, metrics}, + {group, stats}, + {group, alarms}]. + +groups() -> + [ + {pubsub, [sequence], [subscribe_unsubscribe, + publish, pubsub, + t_shared_subscribe, + dispatch_with_no_sub, + 'pubsub#', 'pubsub+']}, + {session, [sequence], [start_session]}, + {metrics, [sequence], [inc_dec_metric]}, + {stats, [sequence], [set_get_stat]}, + {alarms, [sequence], [set_alarms]} + ]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +%%-------------------------------------------------------------------- +%% PubSub Test +%%-------------------------------------------------------------------- + +subscribe_unsubscribe(_) -> + ok = emqx:subscribe(<<"topic">>, <<"clientId">>), + ok = emqx:subscribe(<<"topic/1">>, <<"clientId">>, #{ qos => 1 }), + ok = emqx:subscribe(<<"topic/2">>, <<"clientId">>, #{ qos => 2 }), + true = emqx:subscribed(<<"clientId">>, <<"topic">>), + Topics = emqx:topics(), + lists:foreach(fun(Topic) -> + ?assert(lists:member(Topic, Topics)) + end, Topics), + ok = emqx:unsubscribe(<<"topic">>), + ok = emqx:unsubscribe(<<"topic/1">>), + ok = emqx:unsubscribe(<<"topic/2">>). + +publish(_) -> + Msg = emqx_message:make(ct, <<"test/pubsub">>, <<"hello">>), + ok = emqx:subscribe(<<"test/+">>), + timer:sleep(10), + emqx:publish(Msg), + ?assert(receive {dispatch, <<"test/+">>, Msg} -> true after 5 -> false end). + +dispatch_with_no_sub(_) -> + Msg = emqx_message:make(ct, <<"no_subscribers">>, <<"hello">>), + Delivery = #delivery{sender = self(), message = Msg, results = []}, + ?assertEqual(Delivery, emqx_broker:route([{<<"no_subscribers">>, node()}], Delivery)). + +pubsub(_) -> + true = emqx:is_running(node()), + Self = self(), + Subscriber = <<"clientId">>, + ok = emqx:subscribe(<<"a/b/c">>, Subscriber, #{ qos => 1 }), + #{qos := 1} = ets:lookup_element(emqx_suboption, {Self, <<"a/b/c">>}, 2), + #{qos := 1} = emqx_broker:get_subopts(Subscriber, <<"a/b/c">>), + true = emqx_broker:set_subopts(<<"a/b/c">>, #{qos => 0}), + #{qos := 0} = emqx_broker:get_subopts(Subscriber, <<"a/b/c">>), + ok = emqx:subscribe(<<"a/b/c">>, Subscriber, #{ qos => 2 }), + %% ct:log("Emq Sub: ~p.~n", [ets:lookup(emqx_suboption, {<<"a/b/c">>, Subscriber})]), + timer:sleep(10), + [Self] = emqx_broker:subscribers(<<"a/b/c">>), + emqx:publish(emqx_message:make(ct, <<"a/b/c">>, <<"hello">>)), + ?assert( + receive {dispatch, <<"a/b/c">>, _ } -> + true; + P -> + ct:log("Receive Message: ~p~n",[P]) + after 2 -> + false + end), + spawn(fun() -> + emqx:subscribe(<<"a/b/c">>), + emqx:subscribe(<<"c/d/e">>), + timer:sleep(10), + emqx:unsubscribe(<<"a/b/c">>) + end), + timer:sleep(20), + emqx:unsubscribe(<<"a/b/c">>). + +t_shared_subscribe(_) -> + emqx:subscribe("$share/group2/topic2"), + emqx:subscribe("$queue/topic3"), + timer:sleep(10), + ct:log("share subscriptions: ~p~n", [emqx:subscriptions(self())]), + ?assertEqual(2, length(emqx:subscriptions(self()))), + emqx:unsubscribe("$share/group2/topic2"), + emqx:unsubscribe("$queue/topic3"), + ?assertEqual(0, length(emqx:subscriptions(self()))). + +'pubsub#'(_) -> + emqx:subscribe(<<"a/#">>), + timer:sleep(10), + emqx:publish(emqx_message:make(ct, <<"a/b/c">>, <<"hello">>)), + ?assert(receive {dispatch, <<"a/#">>, _} -> true after 2 -> false end), + emqx:unsubscribe(<<"a/#">>). + +'pubsub+'(_) -> + emqx:subscribe(<<"a/+/+">>), + timer:sleep(10), + emqx:publish(emqx_message:make(ct, <<"a/b/c">>, <<"hello">>)), + ?assert(receive {dispatch, <<"a/+/+">>, _} -> true after 1 -> false end), + emqx:unsubscribe(<<"a/+/+">>). + +%%-------------------------------------------------------------------- +%% Session Group +%%-------------------------------------------------------------------- +start_session(_) -> + ClientId = <<"clientId">>, + {ok, ClientPid} = emqx_mock_client:start_link(ClientId), + {ok, SessPid} = emqx_mock_client:open_session(ClientPid, ClientId, internal), + Message1 = emqx_message:make(<<"clientId">>, 2, <<"topic">>, <<"hello">>), + emqx_session:publish(SessPid, 1, Message1), + emqx_session:pubrel(SessPid, 2, reasoncode), + emqx_session:subscribe(SessPid, [{<<"topic/session">>, #{qos => 2}}]), + Message2 = emqx_message:make(<<"clientId">>, 1, <<"topic/session">>, <<"test">>), + emqx_session:publish(SessPid, 3, Message2), + emqx_session:unsubscribe(SessPid, [{<<"topic/session">>, []}]), + %% emqx_mock_client:stop(ClientPid). + emqx_mock_client:close_session(ClientPid). + +%%-------------------------------------------------------------------- +%% Broker Group +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Metric Group +%%-------------------------------------------------------------------- +inc_dec_metric(_) -> + emqx_metrics:inc(gauge, 'messages/retained', 10), + emqx_metrics:dec(gauge, 'messages/retained', 10). + +%%-------------------------------------------------------------------- +%% Stats Group +%%-------------------------------------------------------------------- + +set_get_stat(_) -> + emqx_stats:setstat('retained/max', 99), + 99 = emqx_stats:getstat('retained/max'). + +set_alarms(_) -> + AlarmTest = #alarm{id = <<"1">>, severity = error, title="alarm title", summary="alarm summary"}, + emqx_alarm_mgr:set_alarm(AlarmTest), + Alarms = emqx_alarm_mgr:get_alarms(), + ct:log("Alarms Length: ~p ~n", [length(Alarms)]), + ?assertEqual(1, length(Alarms)), + emqx_alarm_mgr:clear_alarm(<<"1">>), + [] = emqx_alarm_mgr:get_alarms(). diff --git a/test/emqx_client_SUITE.erl b/test/emqx_client_SUITE.erl new file mode 100644 index 000000000..13303bfdc --- /dev/null +++ b/test/emqx_client_SUITE.erl @@ -0,0 +1,294 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_client_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-import(lists, [nth/2]). + +-include("emqx_mqtt.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). + +-define(TOPICS, [<<"TopicA">>, <<"TopicA/B">>, <<"Topic/C">>, <<"TopicA/C">>, + <<"/TopicA">>]). + +-define(WILD_TOPICS, [<<"TopicA/+">>, <<"+/C">>, <<"#">>, <<"/#">>, <<"/+">>, + <<"+/+">>, <<"TopicA/#">>]). + +all() -> + [{group, mqttv4}, + {group, mqttv5}]. + +groups() -> + [{mqttv4, [non_parallel_tests], + [basic_test, + will_message_test, + offline_message_queueing_test, + overlapping_subscriptions_test, + %% keepalive_test, + redelivery_on_reconnect_test, + %% subscribe_failure_test, + dollar_topics_test]}, + {mqttv5, [non_parallel_tests], + [request_response, + share_sub_request_topic]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +request_response_exception(QoS) -> + {ok, Client} = emqx_client:start_link([{proto_ver, v5}, + {properties, #{ 'Request-Response-Information' => 0 }}]), + {ok, _} = emqx_client:connect(Client), + ?assertError(no_response_information, + emqx_client:sub_request_topic(Client, QoS, <<"request_topic">>)), + ok = emqx_client:disconnect(Client). + +request_response_per_qos(QoS) -> + {ok, Requester} = emqx_client:start_link([{proto_ver, v5}, + {client_id, <<"requester">>}, + {properties, #{ 'Request-Response-Information' => 1}}]), + {ok, _} = emqx_client:connect(Requester), + {ok, Responser} = emqx_client:start_link([{proto_ver, v5}, + {client_id, <<"responser">>}, + {properties, #{ + 'Request-Response-Information' => 1}}, + {request_handler, + fun(_Req) -> <<"ResponseTest">> end} + ]), + {ok, _} = emqx_client:connect(Responser), + ok = emqx_client:sub_request_topic(Responser, QoS, <<"request_topic">>), + {ok, <<"ResponseTest">>} = emqx_client:request(Requester, <<"response_topic">>, <<"request_topic">>, <<"request_payload">>, QoS), + ok = emqx_client:set_request_handler(Responser, fun(<<"request_payload">>) -> + <<"ResponseFunctionTest">>; + (_) -> + <<"404">> + end), + {ok, <<"ResponseFunctionTest">>} = emqx_client:request(Requester, <<"response_topic">>, <<"request_topic">>, <<"request_payload">>, QoS), + {ok, <<"404">>} = emqx_client:request(Requester, <<"response_topic">>, <<"request_topic">>, <<"invalid_request">>, QoS), + ok = emqx_client:disconnect(Responser), + ok = emqx_client:disconnect(Requester). + +request_response(_Config) -> + request_response_per_qos(?QOS_2), + request_response_per_qos(?QOS_1), + request_response_per_qos(?QOS_0), + request_response_exception(?QOS_0), + request_response_exception(?QOS_1), + request_response_exception(?QOS_2). + +share_sub_request_topic(_Config) -> + share_sub_request_topic_per_qos(?QOS_2), + share_sub_request_topic_per_qos(?QOS_1), + share_sub_request_topic_per_qos(?QOS_0). + +share_sub_request_topic_per_qos(QoS) -> + application:set_env(?APPLICATION, shared_subscription_strategy, random), + ReqTopic = <<"request-topic">>, + RspTopic = <<"response-topic">>, + Group = <<"g1">>, + Properties = #{ 'Request-Response-Information' => 1}, + Opts = fun(ClientId) -> [{proto_ver, v5}, + {client_id, atom_to_binary(ClientId, utf8)}, + {properties, Properties} + ] end, + {ok, Requester} = emqx_client:start_link(Opts(requester)), + {ok, _} = emqx_client:connect(Requester), + + {ok, Responser1} = emqx_client:start_link([{request_handler, fun(Req) -> <<"1-", Req/binary>> end} | Opts(requester1)]), + {ok, _} = emqx_client:connect(Responser1), + + {ok, Responser2} = emqx_client:start_link([{request_handler, fun(Req) -> <<"2-", Req/binary>> end} | Opts(requester2)]), + {ok, _} = emqx_client:connect(Responser2), + + ok = emqx_client:sub_request_topic(Responser1, QoS, ReqTopic, Group), + ok = emqx_client:sub_request_topic(Responser2, QoS, ReqTopic, Group), + %% Send a request, wait for response, validate response then return responser ID + ReqFun = fun(Req) -> + {ok, Rsp} = emqx_client:request(Requester, RspTopic, ReqTopic, Req, QoS), + case Rsp of + <<"1-", Req/binary>> -> 1; + <<"2-", Req/binary>> -> 2 + end + end, + Ids = lists:map(fun(I) -> ReqFun(integer_to_binary(I)) end, lists:seq(1, 100)), + %% we are testing with random shared-dispatch strategy, + %% fail if not all responsers got a chance to handle requests + ?assertEqual([1, 2], lists:usort(Ids)), + ok = emqx_client:disconnect(Responser1), + ok = emqx_client:disconnect(Responser2), + ok = emqx_client:disconnect(Requester). + +receive_messages(Count) -> + receive_messages(Count, []). + +receive_messages(0, Msgs) -> + Msgs; +receive_messages(Count, Msgs) -> + receive + {publish, Msg} -> + receive_messages(Count-1, [Msg|Msgs]); + _Other -> + receive_messages(Count, Msgs) + after 10 -> + Msgs + end. + +basic_test(_Config) -> + Topic = nth(1, ?TOPICS), + ct:print("Basic test starting"), + {ok, C} = emqx_client:start_link(), + {ok, _} = emqx_client:connect(C), + {ok, _, [1]} = emqx_client:subscribe(C, Topic, qos1), + {ok, _, [2]} = emqx_client:subscribe(C, Topic, qos2), + {ok, _} = emqx_client:publish(C, Topic, <<"qos 2">>, 2), + {ok, _} = emqx_client:publish(C, Topic, <<"qos 2">>, 2), + {ok, _} = emqx_client:publish(C, Topic, <<"qos 2">>, 2), + ?assertEqual(3, length(receive_messages(3))), + ok = emqx_client:disconnect(C). + +will_message_test(_Config) -> + {ok, C1} = emqx_client:start_link([{clean_start, true}, + {will_topic, nth(3, ?TOPICS)}, + {will_payload, <<"client disconnected">>}, + {keepalive, 2}]), + {ok, _} = emqx_client:connect(C1), + + {ok, C2} = emqx_client:start_link(), + {ok, _} = emqx_client:connect(C2), + + {ok, _, [2]} = emqx_client:subscribe(C2, nth(3, ?TOPICS), 2), + timer:sleep(10), + ok = emqx_client:stop(C1), + timer:sleep(5), + ?assertEqual(1, length(receive_messages(1))), + ok = emqx_client:disconnect(C2), + ct:print("Will message test succeeded"). + +offline_message_queueing_test(_) -> + {ok, C1} = emqx_client:start_link([{clean_start, false}, + {client_id, <<"c1">>}]), + {ok, _} = emqx_client:connect(C1), + + {ok, _, [2]} = emqx_client:subscribe(C1, nth(6, ?WILD_TOPICS), 2), + ok = emqx_client:disconnect(C1), + {ok, C2} = emqx_client:start_link([{clean_start, true}, + {client_id, <<"c2">>}]), + {ok, _} = emqx_client:connect(C2), + + ok = emqx_client:publish(C2, nth(2, ?TOPICS), <<"qos 0">>, 0), + {ok, _} = emqx_client:publish(C2, nth(3, ?TOPICS), <<"qos 1">>, 1), + {ok, _} = emqx_client:publish(C2, nth(4, ?TOPICS), <<"qos 2">>, 2), + timer:sleep(10), + emqx_client:disconnect(C2), + {ok, C3} = emqx_client:start_link([{clean_start, false}, + {client_id, <<"c1">>}]), + {ok, _} = emqx_client:connect(C3), + + timer:sleep(10), + emqx_client:disconnect(C3), + ?assertEqual(3, length(receive_messages(3))). + +overlapping_subscriptions_test(_) -> + {ok, C} = emqx_client:start_link([]), + {ok, _} = emqx_client:connect(C), + + {ok, _, [2, 1]} = emqx_client:subscribe(C, [{nth(7, ?WILD_TOPICS), 2}, + {nth(1, ?WILD_TOPICS), 1}]), + timer:sleep(10), + {ok, _} = emqx_client:publish(C, nth(4, ?TOPICS), <<"overlapping topic filters">>, 2), + timer:sleep(10), + + Num = length(receive_messages(2)), + ?assert(lists:member(Num, [1, 2])), + if + Num == 1 -> + ct:print("This server is publishing one message for all + matching overlapping subscriptions, not one for each."); + Num == 2 -> + ct:print("This server is publishing one message per each + matching overlapping subscription."); + true -> ok + end, + emqx_client:disconnect(C). + +%% keepalive_test(_) -> +%% ct:print("Keepalive test starting"), +%% {ok, C1, _} = emqx_client:start_link([{clean_start, true}, +%% {keepalive, 5}, +%% {will_flag, true}, +%% {will_topic, nth(5, ?TOPICS)}, +%% %% {will_qos, 2}, +%% {will_payload, <<"keepalive expiry">>}]), +%% ok = emqx_client:pause(C1), +%% {ok, C2, _} = emqx_client:start_link([{clean_start, true}, +%% {keepalive, 0}]), +%% {ok, _, [2]} = emqx_client:subscribe(C2, nth(5, ?TOPICS), 2), +%% ok = emqx_client:disconnect(C2), +%% ?assertEqual(1, length(receive_messages(1))), +%% ct:print("Keepalive test succeeded"). + +redelivery_on_reconnect_test(_) -> + ct:print("Redelivery on reconnect test starting"), + {ok, C1} = emqx_client:start_link([{clean_start, false}, + {client_id, <<"c">>}]), + {ok, _} = emqx_client:connect(C1), + + {ok, _, [2]} = emqx_client:subscribe(C1, nth(7, ?WILD_TOPICS), 2), + timer:sleep(10), + ok = emqx_client:pause(C1), + {ok, _} = emqx_client:publish(C1, nth(2, ?TOPICS), <<>>, + [{qos, 1}, {retain, false}]), + {ok, _} = emqx_client:publish(C1, nth(4, ?TOPICS), <<>>, + [{qos, 2}, {retain, false}]), + timer:sleep(10), + ok = emqx_client:disconnect(C1), + ?assertEqual(0, length(receive_messages(2))), + {ok, C2} = emqx_client:start_link([{clean_start, false}, + {client_id, <<"c">>}]), + {ok, _} = emqx_client:connect(C2), + + timer:sleep(10), + ok = emqx_client:disconnect(C2), + ?assertEqual(2, length(receive_messages(2))). + +%% subscribe_failure_test(_) -> +%% ct:print("Subscribe failure test starting"), +%% {ok, C, _} = emqx_client:start_link([]), +%% {ok, _, [2]} = emqx_client:subscribe(C, <<"$SYS/#">>, 2), +%% timer:sleep(10), +%% ct:print("Subscribe failure test succeeded"). + +dollar_topics_test(_) -> + ct:print("$ topics test starting"), + {ok, C} = emqx_client:start_link([{clean_start, true}, + {keepalive, 0}]), + {ok, _} = emqx_client:connect(C), + + {ok, _, [1]} = emqx_client:subscribe(C, nth(6, ?WILD_TOPICS), 1), + {ok, _} = emqx_client:publish(C, << <<"$">>/binary, (nth(2, ?TOPICS))/binary>>, + <<"test">>, [{qos, 1}, {retain, false}]), + timer:sleep(10), + ?assertEqual(0, length(receive_messages(1))), + ok = emqx_client:disconnect(C), + ct:print("$ topics test succeeded"). diff --git a/test/emqx_cm_SUITE.erl b/test/emqx_cm_SUITE.erl new file mode 100644 index 000000000..3d879a476 --- /dev/null +++ b/test/emqx_cm_SUITE.erl @@ -0,0 +1,70 @@ + +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_cm_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> [{group, cm}]. + +groups() -> + [{cm, [non_parallel_tests], + [t_get_set_conn_attrs, + t_get_set_conn_stats, + t_lookup_conn_pid]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +init_per_testcase(_TestCase, Config) -> + register_connection(), + Config. + +end_per_testcase(_TestCase, _Config) -> + unregister_connection(), + ok. + +t_get_set_conn_attrs(_) -> + ?assert(emqx_cm:set_conn_attrs(<<"conn1">>, [{port, 8080}, {ip, "192.168.0.1"}])), + ?assert(emqx_cm:set_conn_attrs(<<"conn2">>, self(), [{port, 8080}, {ip, "192.168.0.2"}])), + ?assertEqual([{port, 8080}, {ip, "192.168.0.1"}], emqx_cm:get_conn_attrs(<<"conn1">>)), + ?assertEqual([{port, 8080}, {ip, "192.168.0.2"}], emqx_cm:get_conn_attrs(<<"conn2">>, self())). + +t_get_set_conn_stats(_) -> + ?assert(emqx_cm:set_conn_stats(<<"conn1">>, [{count, 1}, {max, 2}])), + ?assert(emqx_cm:set_conn_stats(<<"conn2">>, self(), [{count, 1}, {max, 2}])), + ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats(<<"conn1">>)), + ?assertEqual([{count, 1}, {max, 2}], emqx_cm:get_conn_stats(<<"conn2">>, self())). + +t_lookup_conn_pid(_) -> + ?assertEqual(ok, emqx_cm:register_connection(<<"conn1">>, self())), + ?assertEqual(self(), emqx_cm:lookup_conn_pid(<<"conn1">>)). + +register_connection() -> + ?assertEqual(ok, emqx_cm:register_connection(<<"conn1">>)), + ?assertEqual(ok, emqx_cm:register_connection(<<"conn2">>, self())). + +unregister_connection() -> + ?assertEqual(ok, emqx_cm:unregister_connection(<<"conn1">>)), + ?assertEqual(ok, emqx_cm:unregister_connection(<<"conn2">>, self())). diff --git a/test/emqx_ct_broker_helpers.erl b/test/emqx_ct_broker_helpers.erl new file mode 100644 index 000000000..62a91df54 --- /dev/null +++ b/test/emqx_ct_broker_helpers.erl @@ -0,0 +1,126 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_ct_broker_helpers). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(APP, emqx). + +-define(MQTT_SSL_TWOWAY, [{cacertfile, "certs/cacert.pem"}, + {verify, verify_peer}, + {fail_if_no_peer_cert, true}]). + +-define(MQTT_SSL_CLIENT, [{keyfile, "certs/client-key.pem"}, + {cacertfile, "certs/cacert.pem"}, + {certfile, "certs/client-cert.pem"}]). + +-define(CIPHERS, [{ciphers, + ["ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", + "ECDHE-ECDSA-AES256-SHA384", + "ECDHE-RSA-AES256-SHA384","ECDHE-ECDSA-DES-CBC3-SHA", + "ECDH-ECDSA-AES256-GCM-SHA384", + "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384","ECDH-RSA-AES256-SHA384", + "DHE-DSS-AES256-GCM-SHA384","DHE-DSS-AES256-SHA256", + "AES256-GCM-SHA384","AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", + "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA256", + "ECDHE-RSA-AES128-SHA256", + "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-RSA-AES128-GCM-SHA256", + "ECDH-ECDSA-AES128-SHA256","ECDH-RSA-AES128-SHA256", + "DHE-DSS-AES128-GCM-SHA256","DHE-DSS-AES128-SHA256", + "AES128-GCM-SHA256","AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA","ECDHE-RSA-AES256-SHA", + "DHE-DSS-AES256-SHA","ECDH-ECDSA-AES256-SHA", + "ECDH-RSA-AES256-SHA","AES256-SHA", + "ECDHE-ECDSA-AES128-SHA","ECDHE-RSA-AES128-SHA", + "DHE-DSS-AES128-SHA","ECDH-ECDSA-AES128-SHA", + "ECDH-RSA-AES128-SHA","AES128-SHA"]}]). + +run_setup_steps() -> + NewConfig = generate_config(), + lists:foreach(fun set_app_env/1, NewConfig), + application:ensure_all_started(?APP). + +run_teardown_steps() -> + ?APP:shutdown(). + +generate_config() -> + Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]), + Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]), + cuttlefish_generator:map(Schema, Conf). + +get_base_dir(Module) -> + {file, Here} = code:is_loaded(Module), + filename:dirname(filename:dirname(Here)). + +get_base_dir() -> + get_base_dir(?MODULE). + +local_path(Components, Module) -> + filename:join([get_base_dir(Module) | Components]). + +local_path(Components) -> + local_path(Components, ?MODULE). + +set_app_env({App, Lists}) -> + lists:foreach(fun({acl_file, _Var}) -> + application:set_env(App, acl_file, local_path(["etc", "acl.conf"])); + ({plugins_loaded_file, _Var}) -> + application:set_env(App, plugins_loaded_file, local_path(["test", "emqx_SUITE_data","loaded_plugins"])); + ({Par, Var}) -> + application:set_env(App, Par, Var) + end, Lists). + +change_opts(SslType) -> + {ok, Listeners} = application:get_env(?APP, listeners), + NewListeners = + lists:foldl(fun({Protocol, Port, Opts} = Listener, Acc) -> + case Protocol of + ssl -> + SslOpts = proplists:get_value(ssl_options, Opts), + Keyfile = local_path(["etc/certs", "key.pem"]), + Certfile = local_path(["etc/certs", "cert.pem"]), + TupleList1 = lists:keyreplace(keyfile, 1, SslOpts, {keyfile, Keyfile}), + TupleList2 = lists:keyreplace(certfile, 1, TupleList1, {certfile, Certfile}), + TupleList3 = + case SslType of + ssl_twoway-> + CAfile = local_path(["etc", proplists:get_value(cacertfile, ?MQTT_SSL_TWOWAY)]), + MutSslList = lists:keyreplace(cacertfile, 1, ?MQTT_SSL_TWOWAY, {cacertfile, CAfile}), + lists:merge(TupleList2, MutSslList); + _ -> + lists:filter(fun ({cacertfile, _}) -> false; + ({verify, _}) -> false; + ({fail_if_no_peer_cert, _}) -> false; + (_) -> true + end, TupleList2) + end, + [{Protocol, Port, lists:keyreplace(ssl_options, 1, Opts, {ssl_options, TupleList3})} | Acc]; + _ -> + [Listener | Acc] + end + end, [], Listeners), + application:set_env(?APP, listeners, NewListeners). + +client_ssl_twoway() -> + [{Key, local_path(["etc", File])} || {Key, File} <- ?MQTT_SSL_CLIENT] ++ ?CIPHERS. + +client_ssl() -> + ?CIPHERS ++ [{reuse_sessions, true}]. diff --git a/test/emqx_ct_helpers.erl b/test/emqx_ct_helpers.erl new file mode 100644 index 000000000..c61c5be6e --- /dev/null +++ b/test/emqx_ct_helpers.erl @@ -0,0 +1,22 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_ct_helpers). + +-export([ensure_mnesia_stopped/0]). + +ensure_mnesia_stopped() -> + ekka_mnesia:ensure_stopped(), + ekka_mnesia:delete_schema(). + diff --git a/test/emqx_frame_SUITE.erl b/test/emqx_frame_SUITE.erl new file mode 100644 index 000000000..a4e64c7d0 --- /dev/null +++ b/test/emqx_frame_SUITE.erl @@ -0,0 +1,429 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_frame_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_mqtt.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +-import(emqx_frame, [serialize/1, serialize/2]). + +all() -> + [{group, connect}, + {group, connack}, + {group, publish}, + {group, puback}, + {group, subscribe}, + {group, suback}, + {group, unsubscribe}, + {group, unsuback}, + {group, ping}, + {group, disconnect}, + {group, auth}]. + +groups() -> + [{connect, [parallel], + [serialize_parse_connect, + serialize_parse_v3_connect, + serialize_parse_v4_connect, + serialize_parse_v5_connect, + serialize_parse_connect_without_clientid, + serialize_parse_connect_with_will, + serialize_parse_bridge_connect]}, + {connack, [parallel], + [serialize_parse_connack, + serialize_parse_connack_v5]}, + {publish, [parallel], + [serialize_parse_qos0_publish, + serialize_parse_qos1_publish, + serialize_parse_qos2_publish, + serialize_parse_publish_v5]}, + {puback, [parallel], + [serialize_parse_puback, + serialize_parse_puback_v5, + serialize_parse_pubrec, + serialize_parse_pubrec_v5, + serialize_parse_pubrel, + serialize_parse_pubrel_v5, + serialize_parse_pubcomp, + serialize_parse_pubcomp_v5]}, + {subscribe, [parallel], + [serialize_parse_subscribe, + serialize_parse_subscribe_v5]}, + {suback, [parallel], + [serialize_parse_suback, + serialize_parse_suback_v5]}, + {unsubscribe, [parallel], + [serialize_parse_unsubscribe, + serialize_parse_unsubscribe_v5]}, + {unsuback, [parallel], + [serialize_parse_unsuback, + serialize_parse_unsuback_v5]}, + {ping, [parallel], + [serialize_parse_pingreq, + serialize_parse_pingresp]}, + {disconnect, [parallel], + [serialize_parse_disconnect, + serialize_parse_disconnect_v5]}, + {auth, [parallel], + [serialize_parse_auth_v5]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +serialize_parse_connect(_) -> + Packet1 = ?CONNECT_PACKET(#mqtt_packet_connect{}), + ?assertEqual({ok, Packet1, <<>>}, parse_serialize(Packet1)), + Packet2 = ?CONNECT_PACKET(#mqtt_packet_connect{ + client_id = <<"clientId">>, + will_qos = ?QOS_1, + will_flag = true, + will_retain = true, + will_topic = <<"will">>, + will_payload = <<"bye">>, + clean_start = true}), + ?assertEqual({ok, Packet2, <<>>}, parse_serialize(Packet2)). + +serialize_parse_v3_connect(_) -> + Bin = <<16,37,0,6,77,81,73,115,100,112,3,2,0,60,0,23,109,111,115, + 113,112,117, 98,47,49,48,52,53,49,45,105,77,97,99,46,108, + 111,99,97>>, + Packet = ?CONNECT_PACKET( + #mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V3, + proto_name = <<"MQIsdp">>, + client_id = <<"mosqpub/10451-iMac.loca">>, + clean_start = true, + keepalive = 60}), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_v4_connect(_) -> + Bin = <<16,35,0,4,77,81,84,84,4,2,0,60,0,23,109,111,115,113,112,117, + 98,47,49,48,52,53,49,45,105,77,97,99,46,108,111,99,97>>, + Packet = ?CONNECT_PACKET(#mqtt_packet_connect{proto_ver = 4, + proto_name = <<"MQTT">>, + client_id = <<"mosqpub/10451-iMac.loca">>, + clean_start = true, + keepalive = 60}), + ?assertEqual(Bin, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_v5_connect(_) -> + Props = #{'Session-Expiry-Interval' => 60, + 'Receive-Maximum' => 100, + 'Maximum-QoS' => ?QOS_2, + 'Retain-Available' => 1, + 'Maximum-Packet-Size' => 1024, + 'Topic-Alias-Maximum' => 10, + 'Request-Response-Information' => 1, + 'Request-Problem-Information' => 1, + 'Authentication-Method' => <<"oauth2">>, + 'Authentication-Data' => <<"33kx93k">>}, + + WillProps = #{'Will-Delay-Interval' => 60, + 'Payload-Format-Indicator' => 1, + 'Message-Expiry-Interval' => 60, + 'Content-Type' => <<"text/json">>, + 'Response-Topic' => <<"topic">>, + 'Correlation-Data' => <<"correlateid">>, + 'User-Property' => [{<<"k">>, <<"v">>}]}, + Packet = ?CONNECT_PACKET( + #mqtt_packet_connect{proto_name = <<"MQTT">>, + proto_ver = ?MQTT_PROTO_V5, + is_bridge = false, + clean_start = true, + client_id = <<>>, + will_flag = true, + will_qos = ?QOS_1, + will_retain = false, + keepalive = 60, + properties = Props, + will_props = WillProps, + will_topic = <<"topic">>, + will_payload = <<>>, + username = <<"device:1">>, + password = <<"passwd">>}), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_connect_without_clientid(_) -> + Bin = <<16,12,0,4,77,81,84,84,4,2,0,60,0,0>>, + Packet = ?CONNECT_PACKET(#mqtt_packet_connect{proto_ver = 4, + proto_name = <<"MQTT">>, + client_id = <<>>, + clean_start = true, + keepalive = 60}), + ?assertEqual(Bin, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_connect_with_will(_) -> + Bin = <<16,67,0,6,77,81,73,115,100,112,3,206,0,60,0,23,109,111,115,113,112, + 117,98,47,49,48,52,53,50,45,105,77,97,99,46,108,111,99,97,0,5,47,119, + 105,108,108,0,7,119,105,108,108,109,115,103,0,4,116,101,115,116,0,6, + 112,117,98,108,105,99>>, + Packet = #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT}, + variable = #mqtt_packet_connect{proto_ver = ?MQTT_PROTO_V3, + proto_name = <<"MQIsdp">>, + client_id = <<"mosqpub/10452-iMac.loca">>, + clean_start = true, + keepalive = 60, + will_retain = false, + will_qos = ?QOS_1, + will_flag = true, + will_topic = <<"/will">>, + will_payload = <<"willmsg">>, + username = <<"test">>, + password = <<"public">>}}, + ?assertEqual(Bin, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_bridge_connect(_) -> + Bin = <<16,86,0,6,77,81,73,115,100,112,131,44,0,60,0,19,67,95,48,48,58,48,67, + 58,50,57,58,50,66,58,55,55,58,53,50,0,48,36,83,89,83,47,98,114,111,107, + 101,114,47,99,111,110,110,101,99,116,105,111,110,47,67,95,48,48,58,48, + 67,58,50,57,58,50,66,58,55,55,58,53,50,47,115,116,97,116,101,0,1,48>>, + Topic = <<"$SYS/broker/connection/C_00:0C:29:2B:77:52/state">>, + Packet = #mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT}, + variable = #mqtt_packet_connect{client_id = <<"C_00:0C:29:2B:77:52">>, + proto_ver = 16#03, + proto_name = <<"MQIsdp">>, + is_bridge = true, + will_retain = true, + will_qos = ?QOS_1, + will_flag = true, + clean_start = false, + keepalive = 60, + will_topic = Topic, + will_payload = <<"0">>}}, + ?assertEqual(Bin, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_connack(_) -> + Packet = ?CONNACK_PACKET(?RC_SUCCESS), + ?assertEqual(<<32,2,0,0>>, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_connack_v5(_) -> + Props = #{'Session-Expiry-Interval' => 60, + 'Receive-Maximum' => 100, + 'Maximum-QoS' => ?QOS_2, + 'Retain-Available' => 1, + 'Maximum-Packet-Size' => 1024, + 'Assigned-Client-Identifier' => <<"id">>, + 'Topic-Alias-Maximum' => 10, + 'Reason-String' => <<>>, + 'Wildcard-Subscription-Available' => 1, + 'Subscription-Identifier-Available' => 1, + 'Shared-Subscription-Available' => 1, + 'Server-Keep-Alive' => 60, + 'Response-Information' => <<"response">>, + 'Server-Reference' => <<"192.168.1.10">>, + 'Authentication-Method' => <<"oauth2">>, + 'Authentication-Data' => <<"33kx93k">>}, + Packet = ?CONNACK_PACKET(?RC_SUCCESS, 0, Props), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_qos0_publish(_) -> + Bin = <<48,14,0,7,120,120,120,47,121,121,121,104,101,108,108,111>>, + Packet = #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + dup = false, + qos = ?QOS_0, + retain = false}, + variable = #mqtt_packet_publish{topic_name = <<"xxx/yyy">>, + packet_id = undefined}, + payload = <<"hello">>}, + ?assertEqual(Bin, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_qos1_publish(_) -> + Bin = <<50,13,0,5,97,47,98,47,99,0,1,104,97,104,97>>, + Packet = #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + dup = false, + qos = ?QOS_1, + retain = false}, + variable = #mqtt_packet_publish{topic_name = <<"a/b/c">>, + packet_id = 1}, + payload = <<"haha">>}, + ?assertEqual(Bin, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_qos2_publish(_) -> + Packet = ?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 1, payload()), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_publish_v5(_) -> + Props = #{'Payload-Format-Indicator' => 1, + 'Message-Expiry-Interval' => 60, + 'Topic-Alias' => 16#AB, + 'Response-Topic' => <<"reply">>, + 'Correlation-Data' => <<"correlation-id">>, + 'Subscription-Identifier' => 1, + 'Content-Type' => <<"text/json">>}, + Packet = ?PUBLISH_PACKET(?QOS_1, <<"$share/group/topic">>, 1, Props, <<"payload">>), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_puback(_) -> + Packet = ?PUBACK_PACKET(1), + ?assertEqual(<<64,2,0,1>>, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_puback_v5(_) -> + Packet = ?PUBACK_PACKET(16, ?RC_SUCCESS, #{'Reason-String' => <<"success">>}), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_pubrec(_) -> + Packet = ?PUBREC_PACKET(1), + ?assertEqual(<<5:4,0:4,2,0,1>>, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_pubrec_v5(_) -> + Packet = ?PUBREC_PACKET(16, ?RC_SUCCESS, #{'Reason-String' => <<"success">>}), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_pubrel(_) -> + Packet = ?PUBREL_PACKET(1), + ?assertEqual(<<6:4,2:4,2,0,1>>, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_pubrel_v5(_) -> + Packet = ?PUBREL_PACKET(16, ?RC_SUCCESS, #{'Reason-String' => <<"success">>}), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_pubcomp(_) -> + Packet = ?PUBCOMP_PACKET(1), + ?assertEqual(<<7:4,0:4,2,0,1>>, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_pubcomp_v5(_) -> + Packet = ?PUBCOMP_PACKET(16, ?RC_SUCCESS, #{'Reason-String' => <<"success">>}), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_subscribe(_) -> + %% SUBSCRIBE(Q1, R0, D0, PacketId=2, TopicTable=[{<<"TopicA">>,2}]) + Bin = <<130,11,0,2,0,6,84,111,112,105,99,65,2>>, + TopicOpts = #{ nl => 0 , rap => 0, rc => 0, + rh => 0, qos => 2 }, + TopicFilters = [{<<"TopicA">>, TopicOpts}], + Packet = ?SUBSCRIBE_PACKET(2, TopicFilters), + ?assertEqual(Bin, iolist_to_binary(serialize(Packet))), + ct:log("Bin: ~p, Packet: ~p ~n", [Packet, parse(Bin)]), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_subscribe_v5(_) -> + TopicFilters = [{<<"TopicQos0">>, #{rh => 1, qos => ?QOS_2, rap => 0, nl => 0, rc => 0}}, + {<<"TopicQos1">>, #{rh => 1, qos => ?QOS_2, rap => 0, nl => 0, rc => 0}}], + Packet = ?SUBSCRIBE_PACKET(3, #{'Subscription-Identifier' => 16#FFFFFFF}, + TopicFilters), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_suback(_) -> + Packet = ?SUBACK_PACKET(10, [?QOS_0, ?QOS_1, 128]), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_suback_v5(_) -> + Packet = ?SUBACK_PACKET(1, #{'Reason-String' => <<"success">>, + 'User-Property' => [{<<"key">>, <<"value">>}]}, + [?QOS_0, ?QOS_1, 128]), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + + +serialize_parse_unsubscribe(_) -> + %% UNSUBSCRIBE(Q1, R0, D0, PacketId=2, TopicTable=[<<"TopicA">>]) + Packet = ?UNSUBSCRIBE_PACKET(2, [<<"TopicA">>]), + Bin = <<162,10,0,2,0,6,84,111,112,105,99,65>>, + ?assertEqual(Bin, iolist_to_binary(serialize(Packet))), + ?assertEqual({ok, Packet, <<>>}, parse(Bin)). + +serialize_parse_unsubscribe_v5(_) -> + Props = #{'User-Property' => [{<<"key">>, <<"val">>}]}, + Packet = ?UNSUBSCRIBE_PACKET(10, Props, [<<"Topic1">>, <<"Topic2">>]), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_unsuback(_) -> + Packet = ?UNSUBACK_PACKET(10), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_unsuback_v5(_) -> + Packet = ?UNSUBACK_PACKET(10, #{'Reason-String' => <<"Not authorized">>, + 'User-Property' => [{<<"key">>, <<"val">>}]}, + [16#87, 16#87, 16#87]), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_pingreq(_) -> + PingReq = ?PACKET(?PINGREQ), + ?assertEqual({ok, PingReq, <<>>}, parse_serialize(PingReq)). + +serialize_parse_pingresp(_) -> + PingResp = ?PACKET(?PINGRESP), + ?assertEqual({ok, PingResp, <<>>}, parse_serialize(PingResp)). + +parse_disconnect(_) -> + ?assertEqual({ok, ?DISCONNECT_PACKET(?RC_SUCCESS), <<>>}, parse(<<224, 0>>)). + +serialize_parse_disconnect(_) -> + Packet = ?DISCONNECT_PACKET(?RC_SUCCESS), + ?assertEqual({ok, Packet, <<>>}, parse_serialize(Packet)). + +serialize_parse_disconnect_v5(_) -> + Packet = ?DISCONNECT_PACKET(?RC_SUCCESS, + #{'Session-Expiry-Interval' => 60, + 'Reason-String' => <<"server_moved">>, + 'Server-Reference' => <<"192.168.1.10">>}), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +serialize_parse_auth_v5(_) -> + Packet = ?AUTH_PACKET(?RC_SUCCESS, + #{'Authentication-Method' => <<"oauth2">>, + 'Authentication-Data' => <<"3zekkd">>, + 'Reason-String' => <<"success">>, + 'User-Property' => [{<<"key">>, <<"val">>}]}), + ?assertEqual({ok, Packet, <<>>}, + parse_serialize(Packet, #{version => ?MQTT_PROTO_V5})). + +parse_serialize(Packet) -> + parse(iolist_to_binary(serialize(Packet))). + +parse_serialize(Packet, Opts) when is_map(Opts) -> + parse(iolist_to_binary(serialize(Packet, Opts)), Opts). + +parse(Bin) -> + parse(Bin, #{}). + +parse(Bin, Opts) when is_map(Opts) -> + emqx_frame:parse(Bin, emqx_frame:initial_state(Opts)). + +payload() -> + iolist_to_binary(["payload." || _I <- lists:seq(1, 1000)]). diff --git a/test/emqx_gc_SUITE.erl b/test/emqx_gc_SUITE.erl new file mode 100644 index 000000000..22d7cd584 --- /dev/null +++ b/test/emqx_gc_SUITE.erl @@ -0,0 +1,57 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_gc_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [t_init, t_run, t_info, t_reset]. + +t_init(_) -> + ?assertEqual(undefined, emqx_gc:init(false)), + GC1 = emqx_gc:init(#{count => 10, bytes => 0}), + ?assertEqual(#{cnt => {10, 10}}, emqx_gc:info(GC1)), + GC2 = emqx_gc:init(#{count => 0, bytes => 10}), + ?assertEqual(#{oct => {10, 10}}, emqx_gc:info(GC2)), + GC3 = emqx_gc:init(#{count => 10, bytes => 10}), + ?assertEqual(#{cnt => {10, 10}, oct => {10, 10}}, emqx_gc:info(GC3)). + +t_run(_) -> + GC = emqx_gc:init(#{count => 10, bytes => 10}), + ?assertEqual({true, GC}, emqx_gc:run(1, 1000, GC)), + ?assertEqual({true, GC}, emqx_gc:run(1000, 1, GC)), + {false, GC1} = emqx_gc:run(1, 1, GC), + ?assertEqual(#{cnt => {10, 9}, oct => {10, 9}}, emqx_gc:info(GC1)), + {false, GC2} = emqx_gc:run(2, 2, GC1), + ?assertEqual(#{cnt => {10, 7}, oct => {10, 7}}, emqx_gc:info(GC2)), + {false, GC3} = emqx_gc:run(3, 3, GC2), + ?assertEqual(#{cnt => {10, 4}, oct => {10, 4}}, emqx_gc:info(GC3)), + ?assertEqual({true, GC}, emqx_gc:run(4, 4, GC3)). + +t_info(_) -> + ?assertEqual(undefined, emqx_gc:info(undefined)), + GC = emqx_gc:init(#{count => 10, bytes => 0}), + ?assertEqual(#{cnt => {10, 10}}, emqx_gc:info(GC)). + +t_reset(_) -> + ?assertEqual(undefined, emqx_gc:reset(undefined)), + GC = emqx_gc:init(#{count => 10, bytes => 10}), + {false, GC1} = emqx_gc:run(5, 5, GC), + ?assertEqual(#{cnt => {10, 5}, oct => {10, 5}}, emqx_gc:info(GC1)), + ?assertEqual(GC, emqx_gc:reset(GC1)). + diff --git a/test/emqx_guid_SUITE.erl b/test/emqx_guid_SUITE.erl new file mode 100644 index 000000000..cb2e6543c --- /dev/null +++ b/test/emqx_guid_SUITE.erl @@ -0,0 +1,40 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_guid_SUITE). + +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-compile(nowarn_export_all). + +all() -> [t_guid_gen, t_guid_hexstr, t_guid_base62]. + +t_guid_gen(_) -> + Guid1 = emqx_guid:gen(), + Guid2 = emqx_guid:gen(), + <<_:128>> = Guid1, + true = (Guid2 >= Guid1), + {Ts1, _, 0} = emqx_guid:new(), + Ts2 = emqx_guid:timestamp(emqx_guid:gen()), + true = Ts2 > Ts1. + +t_guid_hexstr(_) -> + Guid = emqx_guid:gen(), + ?assertEqual(Guid, emqx_guid:from_hexstr(emqx_guid:to_hexstr(Guid))). + +t_guid_base62(_) -> + Guid = emqx_guid:gen(), + ?assertEqual(Guid, emqx_guid:from_base62(emqx_guid:to_base62(Guid))). + diff --git a/test/emqx_hooks_SUITE.erl b/test/emqx_hooks_SUITE.erl new file mode 100644 index 000000000..1961d0e02 --- /dev/null +++ b/test/emqx_hooks_SUITE.erl @@ -0,0 +1,94 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_hooks_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + [add_delete_hook, run_hooks]. + +add_delete_hook(_) -> + {ok, _} = emqx_hooks:start_link(), + ok = emqx:hook(test_hook, fun ?MODULE:hook_fun1/1, []), + ok = emqx:hook(test_hook, fun ?MODULE:hook_fun2/1, []), + ?assertEqual({error, already_exists}, + emqx:hook(test_hook, fun ?MODULE:hook_fun2/1, [])), + Callbacks = [{callback, {fun ?MODULE:hook_fun1/1, []}, undefined, 0}, + {callback, {fun ?MODULE:hook_fun2/1, []}, undefined, 0}], + ?assertEqual(Callbacks, emqx_hooks:lookup(test_hook)), + ok = emqx:unhook(test_hook, fun ?MODULE:hook_fun1/1), + ok = emqx:unhook(test_hook, fun ?MODULE:hook_fun2/1), + timer:sleep(1000), + ?assertEqual([], emqx_hooks:lookup(test_hook)), + + ok = emqx:hook(emqx_hook, {?MODULE, hook_fun2, []}, 8), + ok = emqx:hook(emqx_hook, {?MODULE, hook_fun1, []}, 9), + Callbacks2 = [{callback, {?MODULE, hook_fun1, []}, undefined, 9}, + {callback, {?MODULE, hook_fun2, []}, undefined, 8}], + ?assertEqual(Callbacks2, emqx_hooks:lookup(emqx_hook)), + ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun1, []}), + ok = emqx:unhook(emqx_hook, {?MODULE, hook_fun2}), + timer:sleep(1000), + ?assertEqual([], emqx_hooks:lookup(emqx_hook)), + ok = emqx_hooks:stop(). + +run_hooks(_) -> + {ok, _} = emqx_hooks:start_link(), + ok = emqx:hook(foldl_hook, fun ?MODULE:hook_fun3/4, [init]), + ok = emqx:hook(foldl_hook, {?MODULE, hook_fun3, [init]}), + ok = emqx:hook(foldl_hook, fun ?MODULE:hook_fun4/4, [init]), + ok = emqx:hook(foldl_hook, fun ?MODULE:hook_fun5/4, [init]), + {stop, [r3, r2]} = emqx:run_hooks(foldl_hook, [arg1, arg2], []), + {ok, []} = emqx:run_hooks(unknown_hook, [], []), + + ok = emqx:hook(foreach_hook, fun ?MODULE:hook_fun6/2, [initArg]), + {error, already_exists} = emqx:hook(foreach_hook, fun ?MODULE:hook_fun6/2, [initArg]), + ok = emqx:hook(foreach_hook, fun ?MODULE:hook_fun7/2, [initArg]), + ok = emqx:hook(foreach_hook, fun ?MODULE:hook_fun8/2, [initArg]), + stop = emqx:run_hooks(foreach_hook, [arg]), + + ok = emqx:hook(foldl_hook2, fun ?MODULE:hook_fun9/2), + ok = emqx:hook(foldl_hook2, {?MODULE, hook_fun10, []}), + {stop, []} = emqx:run_hooks(foldl_hook2, [arg], []), + + ok = emqx:hook(filter1_hook, {?MODULE, hook_fun1, []}, {?MODULE, hook_filter1, []}, 0), + ok = emqx:run_hooks(filter1_hook, [arg]), + + ok = emqx:hook(filter2_hook, {?MODULE, hook_fun2, []}, {?MODULE, hook_filter2, []}), + {ok, []} = emqx:run_hooks(filter2_hook, [arg], []), + + ok = emqx_hooks:stop(). + +hook_fun1([]) -> ok. +hook_fun2([]) -> {ok, []}. + +hook_fun3(arg1, arg2, _Acc, init) -> ok. +hook_fun4(arg1, arg2, Acc, init) -> {ok, [r2 | Acc]}. +hook_fun5(arg1, arg2, Acc, init) -> {stop, [r3 | Acc]}. + +hook_fun6(arg, initArg) -> ok. +hook_fun7(arg, initArg) -> any. +hook_fun8(arg, initArg) -> stop. + +hook_fun9(arg, _Acc) -> any. +hook_fun10(arg, _Acc) -> stop. + +hook_filter1(arg) -> true. +hook_filter2(arg, _Acc) -> true. + diff --git a/test/emqx_inflight_SUITE.erl b/test/emqx_inflight_SUITE.erl new file mode 100644 index 000000000..5e504f4f8 --- /dev/null +++ b/test/emqx_inflight_SUITE.erl @@ -0,0 +1,41 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_inflight_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +all() -> [t_inflight_all]. + +t_inflight_all(_) -> + Empty = emqx_inflight:new(2), + true = emqx_inflight:is_empty(Empty), + 2 = emqx_inflight:max_size(Empty), + false = emqx_inflight:contain(a, Empty), + none = emqx_inflight:lookup(a, Empty), + try emqx_inflight:update(a, 1, Empty) catch + error:Reason -> io:format("Reason: ~w~n", [Reason]) + end, + 0 = emqx_inflight:size(Empty), + Inflight1 = emqx_inflight:insert(a, 1, Empty), + Inflight2 = emqx_inflight:insert(b, 2, Inflight1), + 2 = emqx_inflight:size(Inflight2), + true = emqx_inflight:is_full(Inflight2), + {value, 1} = emqx_inflight:lookup(a, Inflight1), + {value, 2} = emqx_inflight:lookup(a, emqx_inflight:update(a, 2, Inflight1)), + false = emqx_inflight:contain(a, emqx_inflight:delete(a, Inflight1)), + [1, 2] = emqx_inflight:values(Inflight2), + [{a, 1}, {b ,2}] = emqx_inflight:to_list(Inflight2), + [a, b] = emqx_inflight:window(Inflight2). diff --git a/test/emqx_json_SUITE.erl b/test/emqx_json_SUITE.erl new file mode 100644 index 000000000..980d29703 --- /dev/null +++ b/test/emqx_json_SUITE.erl @@ -0,0 +1,37 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_json_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +all() -> [t_decode_encode, t_safe_decode_encode]. + +t_decode_encode(_) -> + JsonText = <<"{\"library\": \"jsx\", \"awesome\": true}">>, + JsonTerm = emqx_json:decode(JsonText), + JsonMaps = #{library => <<"jsx">>, awesome => true}, + JsonMaps = emqx_json:decode(JsonText, [{labels, atom}, return_maps]), + JsonText = emqx_json:encode(JsonTerm, [{space, 1}]). + +t_safe_decode_encode(_) -> + JsonText = <<"{\"library\": \"jsx\", \"awesome\": true}">>, + {ok, JsonTerm} = emqx_json:safe_decode(JsonText), + JsonMaps = #{library => <<"jsx">>, awesome => true}, + {ok, JsonMaps} = emqx_json:safe_decode(JsonText, [{labels, atom}, return_maps]), + {ok, JsonText} = emqx_json:safe_encode(JsonTerm, [{space, 1}]), + BadJsonText = <<"{\"library\", \"awesome\": true}">>, + {error, _} = emqx_json:safe_decode(BadJsonText), + {error, _} = emqx_json:safe_encode({a, {b ,1}}). diff --git a/test/emqttd_mod_SUITE.erl b/test/emqx_keepalive_SUITE.erl similarity index 51% rename from test/emqttd_mod_SUITE.erl rename to test/emqx_keepalive_SUITE.erl index 9935b7424..c4dbd80f2 100644 --- a/test/emqttd_mod_SUITE.erl +++ b/test/emqx_keepalive_SUITE.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,24 +11,32 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_mod_SUITE). +-module(emqx_keepalive_SUITE). -compile(export_all). +-compile(nowarn_export_all). --include("emqttd.hrl"). +all() -> [{group, keepalive}]. -all() -> [mod_subscription_rep]. +groups() -> [{keepalive, [], [t_keepalive]}]. -mod_subscription_rep(_) -> ok. -%% <<"topic/clientId">> = emqttd_mod_subscription:rep( -%% <<"$c">>, <<"clientId">>, <<"topic/$c">>), -%% <<"topic/username">> = emqttd_mod_subscription:rep( -%% <<"$u">>, <<"username">>, <<"topic/$u">>), -%% <<"topic/username/clientId">> = emqttd_mod_subscription:rep( -%% <<"$c">>, <<"clientId">>, emqttd_mod_subscription:rep( -%% <<"$u">>, <<"username">>, <<"topic/$u/$c">>)). - +%%-------------------------------------------------------------------- +%% Keepalive +%%-------------------------------------------------------------------- +t_keepalive(_) -> + {ok, KA} = emqx_keepalive:start(fun() -> {ok, 1} end, 1, {keepalive, timeout}), + [resumed, timeout] = lists:reverse(keepalive_recv(KA, [])). + +keepalive_recv(KA, Acc) -> + receive + {keepalive, timeout} -> + case emqx_keepalive:check(KA) of + {ok, KA1} -> keepalive_recv(KA1, [resumed | Acc]); + {error, timeout} -> [timeout | Acc] + end + after 4000 -> + Acc + end. diff --git a/test/emqttd_lib_SUITE.erl b/test/emqx_lib_SUITE.erl similarity index 64% rename from test/emqttd_lib_SUITE.erl rename to test/emqx_lib_SUITE.erl index dac72e210..12bbd023e 100644 --- a/test/emqttd_lib_SUITE.erl +++ b/test/emqx_lib_SUITE.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,13 +11,13 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_lib_SUITE). +-module(emqx_lib_SUITE). -include_lib("eunit/include/eunit.hrl"). -compile(export_all). +-compile(nowarn_export_all). -define(SOCKOPTS, [ binary, @@ -28,13 +27,13 @@ {nodelay, true} ]). --define(PQ, priority_queue). +-define(PQ, emqx_pqueue). --define(BASE62, emqttd_base62). +-define(BASE62, emqx_base62). all() -> [{group, guid}, {group, opts}, {group, ?PQ}, {group, time}, - {group, base62}]. + {group, node}, {group, base62}]. groups() -> [{guid, [], [guid_gen, guid_hexstr, guid_base62]}, @@ -42,40 +41,41 @@ groups() -> {?PQ, [], [priority_queue_plen, priority_queue_out2]}, {time, [], [time_now_to_]}, + {node, [], [node_is_aliving, node_parse_name]}, {base62, [], [base62_encode]}]. %%-------------------------------------------------------------------- -%% emqttd_guid +%% emqx_guid %%-------------------------------------------------------------------- guid_gen(_) -> - Guid1 = emqttd_guid:gen(), - Guid2 = emqttd_guid:gen(), + Guid1 = emqx_guid:gen(), + Guid2 = emqx_guid:gen(), <<_:128>> = Guid1, true = (Guid2 >= Guid1), - {Ts1, _, 0} = emqttd_guid:new(), - Ts2 = emqttd_guid:timestamp(emqttd_guid:gen()), + {Ts1, _, 0} = emqx_guid:new(), + Ts2 = emqx_guid:timestamp(emqx_guid:gen()), true = Ts2 > Ts1. guid_hexstr(_) -> - Guid = emqttd_guid:gen(), - ?assertEqual(Guid, emqttd_guid:from_hexstr(emqttd_guid:to_hexstr(Guid))). + Guid = emqx_guid:gen(), + ?assertEqual(Guid, emqx_guid:from_hexstr(emqx_guid:to_hexstr(Guid))). guid_base62(_) -> - Guid = emqttd_guid:gen(), - ?assertEqual(Guid, emqttd_guid:from_base62(emqttd_guid:to_base62(Guid))). + Guid = emqx_guid:gen(), + ?assertEqual(Guid, emqx_guid:from_base62(emqx_guid:to_base62(Guid))). %%-------------------------------------------------------------------- -%% emqttd_opts +%% emqx_opts %%-------------------------------------------------------------------- opts_merge(_) -> - Opts = emqttd_misc:merge_opts(?SOCKOPTS, [raw, - binary, - {backlog, 1024}, - {nodelay, false}, - {max_clients, 1024}, - {acceptors, 16}]), + Opts = emqx_misc:merge_opts(?SOCKOPTS, [raw, + binary, + {backlog, 1024}, + {nodelay, false}, + {max_clients, 1024}, + {acceptors, 16}]), 1024 = proplists:get_value(backlog, Opts), 1024 = proplists:get_value(max_clients, Opts), [binary, raw, @@ -135,25 +135,39 @@ priority_queue_out2(_) -> {empty, _Q7} = ?PQ:out(Q6). %%-------------------------------------------------------------------- -%% emqttd_time +%% emqx_time %%-------------------------------------------------------------------- time_now_to_(_) -> - emqttd_time:seed(), - emqttd_time:now_secs(), - emqttd_time:now_ms(). + emqx_time:seed(), + emqx_time:now_secs(), + emqx_time:now_ms(). + +%%-------------------------------------------------------------------- +%% emqx_node +%%-------------------------------------------------------------------- + +node_is_aliving(_) -> + io:format("Node: ~p~n", [node()]), + true = ekka_node:is_aliving(node()), + false = ekka_node:is_aliving('x@127.0.0.1'). + +node_parse_name(_) -> + 'a@127.0.0.1' = ekka_node:parse_name("a@127.0.0.1"), + 'b@127.0.0.1' = ekka_node:parse_name("b"). %%-------------------------------------------------------------------- %% base62 encode decode %%-------------------------------------------------------------------- base62_encode(_) -> - 10 = ?BASE62:decode(?BASE62:encode(10)), - 100 = ?BASE62:decode(?BASE62:encode(100)), - 9999 = ?BASE62:decode(?BASE62:encode(9999)), - 65535 = ?BASE62:decode(?BASE62:encode(65535)), - <> = emqttd_guid:gen(), - <> = emqttd_guid:gen(), - X = ?BASE62:decode(?BASE62:encode(X)), - Y = ?BASE62:decode(?BASE62:encode(Y)). - + <<"10">> = ?BASE62:decode(?BASE62:encode(<<"10">>)), + <<"100">> = ?BASE62:decode(?BASE62:encode(<<"100">>)), + <<"9999">> = ?BASE62:decode(?BASE62:encode(<<"9999">>)), + <<"65535">> = ?BASE62:decode(?BASE62:encode(<<"65535">>)), + <> = emqx_guid:gen(), + <> = emqx_guid:gen(), + X = ?BASE62:decode(?BASE62:encode(X), integer), + Y = ?BASE62:decode(?BASE62:encode(Y), integer), + <<"helloworld">> = ?BASE62:decode(?BASE62:encode("helloworld")), + "helloworld" = ?BASE62:decode(?BASE62:encode("helloworld", string), string). diff --git a/test/emqx_listeners_SUITE.erl b/test/emqx_listeners_SUITE.erl new file mode 100644 index 000000000..17181aa31 --- /dev/null +++ b/test/emqx_listeners_SUITE.erl @@ -0,0 +1,77 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_listeners_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +all() -> + [start_stop_listeners, + restart_listeners]. + +init_per_suite(Config) -> + NewConfig = generate_config(), + application:ensure_all_started(esockd), + application:ensure_all_started(cowboy), + lists:foreach(fun set_app_env/1, NewConfig), + Config. + +end_per_suite(_Config) -> + application:stop(esockd), + application:stop(cowboy). + +start_stop_listeners(_) -> + ok = emqx_listeners:start(), + ok = emqx_listeners:stop(). + +restart_listeners(_) -> + ok = emqx_listeners:start(), + ok = emqx_listeners:stop(), + ok = emqx_listeners:restart(), + ok = emqx_listeners:stop(). + +generate_config() -> + Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]), + Conf = conf_parse:file([local_path(["etc", "gen.emqx.conf"])]), + cuttlefish_generator:map(Schema, Conf). + +set_app_env({App, Lists}) -> + lists:foreach(fun({acl_file, _Var}) -> + application:set_env(App, acl_file, local_path(["etc", "acl.conf"])); + ({plugins_loaded_file, _Var}) -> + application:set_env(App, plugins_loaded_file, local_path(["test", "emqx_SUITE_data","loaded_plugins"])); + ({Par, Var}) -> + application:set_env(App, Par, Var) + end, Lists). + +local_path(Components, Module) -> + filename:join([get_base_dir(Module) | Components]). + +local_path(Components) -> + local_path(Components, ?MODULE). + +get_base_dir(Module) -> + {file, Here} = code:is_loaded(Module), + filename:dirname(filename:dirname(Here)). + +get_base_dir() -> + get_base_dir(?MODULE). diff --git a/test/emqx_message_SUITE.erl b/test/emqx_message_SUITE.erl new file mode 100644 index 000000000..3bb06d14c --- /dev/null +++ b/test/emqx_message_SUITE.erl @@ -0,0 +1,75 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_message_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). + +-include("emqx_mqtt.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [ + message_make, + message_flag, + message_header, + message_format, + message_expired + ]. + +message_make(_) -> + Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), + ?assertEqual(0, Msg#message.qos), + Msg1 = emqx_message:make(<<"clientid">>, ?QOS_2, <<"topic">>, <<"payload">>), + ?assert(is_binary(Msg1#message.id)), + ?assertEqual(2, Msg1#message.qos). + +message_flag(_) -> + Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), + Msg2 = emqx_message:set_flag(retain, false, Msg), + Msg3 = emqx_message:set_flag(dup, Msg2), + ?assert(emqx_message:get_flag(dup, Msg3)), + ?assertNot(emqx_message:get_flag(retain, Msg3)), + Msg4 = emqx_message:unset_flag(dup, Msg3), + Msg5 = emqx_message:unset_flag(retain, Msg4), + ?assertEqual(undefined, emqx_message:get_flag(dup, Msg5, undefined)), + ?assertEqual(undefined, emqx_message:get_flag(retain, Msg5, undefined)), + Msg6 = emqx_message:set_flags(#{dup => true, retain => true}, Msg5), + ?assert(emqx_message:get_flag(dup, Msg6)), + ?assert(emqx_message:get_flag(retain, Msg6)). + +message_header(_) -> + Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), + Msg1 = emqx_message:set_headers(#{a => 1, b => 2}, Msg), + Msg2 = emqx_message:set_header(c, 3, Msg1), + ?assertEqual(1, emqx_message:get_header(a, Msg2)), + ?assertEqual(4, emqx_message:get_header(d, Msg2, 4)). + +message_format(_) -> + io:format("~s", [emqx_message:format(emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>))]). + +message_expired(_) -> + Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), + Msg1 = emqx_message:set_headers(#{'Message-Expiry-Interval' => 1}, Msg), + timer:sleep(500), + ?assertNot(emqx_message:is_expired(Msg1)), + timer:sleep(600), + ?assert(emqx_message:is_expired(Msg1)), + timer:sleep(1000), + Msg2 = emqx_message:update_expiry(Msg1), + ?assertEqual(1, emqx_message:get_header('Message-Expiry-Interval', Msg2)). diff --git a/test/emqx_metrics_SUITE.erl b/test/emqx_metrics_SUITE.erl new file mode 100644 index 000000000..ead7e98ba --- /dev/null +++ b/test/emqx_metrics_SUITE.erl @@ -0,0 +1,58 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_metrics_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_mqtt.hrl"). + +all() -> [t_inc_dec_metrics, t_trans]. + +t_inc_dec_metrics(_) -> + {ok, _} = emqx_metrics:start_link(), + {0, 0} = {emqx_metrics:val('bytes/received'), emqx_metrics:val('messages/retained')}, + emqx_metrics:inc('bytes/received'), + emqx_metrics:inc({counter, 'bytes/received'}, 2), + emqx_metrics:inc(counter, 'bytes/received', 1), + emqx_metrics:inc('bytes/received', 1), + emqx_metrics:inc({gauge, 'messages/retained'}, 2), + emqx_metrics:inc(gauge, 'messages/retained', 2), + {5, 4} = {emqx_metrics:val('bytes/received'), emqx_metrics:val('messages/retained')}, + emqx_metrics:dec(gauge, 'messages/retained'), + emqx_metrics:dec(gauge, 'messages/retained', 1), + 2 = emqx_metrics:val('messages/retained'), + emqx_metrics:set('messages/retained', 3), + 3 = emqx_metrics:val('messages/retained'), + emqx_metrics:received(#mqtt_packet{header = #mqtt_packet_header{type = ?CONNECT}}), + {1, 1} = {emqx_metrics:val('packets/received'), emqx_metrics:val('packets/connect')}, + emqx_metrics:sent(#mqtt_packet{header = #mqtt_packet_header{type = ?CONNACK}}), + {1, 1} = {emqx_metrics:val('packets/sent'), emqx_metrics:val('packets/connack')}. + +t_trans(_) -> + {ok, _} = emqx_metrics:start_link(), + emqx_metrics:trans(inc, 'bytes/received'), + emqx_metrics:trans(inc, {counter, 'bytes/received'}, 2), + emqx_metrics:trans(inc, counter, 'bytes/received', 2), + emqx_metrics:trans(inc, {gauge, 'messages/retained'}, 2), + emqx_metrics:trans(inc, gauge, 'messages/retained', 2), + {0, 0} = {emqx_metrics:val('bytes/received'), emqx_metrics:val('messages/retained')}, + emqx_metrics:commit(), + {5, 4} = {emqx_metrics:val('bytes/received'), emqx_metrics:val('messages/retained')}, + emqx_metrics:trans(dec, gauge, 'messages/retained'), + emqx_metrics:trans(dec, gauge, 'messages/retained', 1), + 4 = emqx_metrics:val('messages/retained'), + emqx_metrics:commit(), + 2 = emqx_metrics:val('messages/retained'). diff --git a/test/emqx_misc_tests.erl b/test/emqx_misc_tests.erl new file mode 100644 index 000000000..92b0973e8 --- /dev/null +++ b/test/emqx_misc_tests.erl @@ -0,0 +1,65 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_misc_tests). +-include_lib("eunit/include/eunit.hrl"). + +-define(SOCKOPTS, [binary, + {packet, raw}, + {reuseaddr, true}, + {backlog, 512}, + {nodelay, true}]). + + +t_merge_opts_test() -> + Opts = emqx_misc:merge_opts(?SOCKOPTS, [raw, + binary, + {backlog, 1024}, + {nodelay, false}, + {max_clients, 1024}, + {acceptors, 16}]), + ?assertEqual(1024, proplists:get_value(backlog, Opts)), + ?assertEqual(1024, proplists:get_value(max_clients, Opts)), + [binary, raw, + {acceptors, 16}, + {backlog, 1024}, + {max_clients, 1024}, + {nodelay, false}, + {packet, raw}, + {reuseaddr, true}] = lists:sort(Opts). + +timer_cancel_flush_test() -> + Timer = emqx_misc:start_timer(0, foo), + ok = emqx_misc:cancel_timer(Timer), + receive {timeout, Timer, foo} -> error(unexpected) + after 0 -> ok + end. + +shutdown_disabled_test() -> + self() ! foo, + ?assertEqual(continue, conn_proc_mng_policy(0)), + receive foo -> ok end, + ?assertEqual(hibernate, conn_proc_mng_policy(0)). + +message_queue_too_long_test() -> + self() ! foo, + self() ! bar, + ?assertEqual({shutdown, message_queue_too_long}, + conn_proc_mng_policy(1)), + receive foo -> ok end, + ?assertEqual(continue, conn_proc_mng_policy(1)), + receive bar -> ok end. + +conn_proc_mng_policy(L) -> + emqx_misc:conn_proc_mng_policy(#{message_queue_len => L}). diff --git a/test/emqx_mock_client.erl b/test/emqx_mock_client.erl new file mode 100644 index 000000000..36d743ef9 --- /dev/null +++ b/test/emqx_mock_client.erl @@ -0,0 +1,99 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mock_client). + +-behaviour(gen_server). + +-export([start_link/1, open_session/3, open_session/4, + close_session/1, stop/1, get_last_message/1]). + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-record(state, {clean_start, client_id, client_pid, last_msg, session_pid}). + +start_link(ClientId) -> + gen_server:start_link(?MODULE, [ClientId], []). + +open_session(ClientPid, ClientId, Zone) -> + open_session(ClientPid, ClientId, Zone, _Attrs = #{}). + +open_session(ClientPid, ClientId, Zone, Attrs0) -> + Attrs1 = default_session_attributes(Zone, ClientId, ClientPid), + Attrs = maps:merge(Attrs1, Attrs0), + gen_server:call(ClientPid, {start_session, ClientPid, ClientId, Attrs}). + +%% close session and terminate the client itself +close_session(ClientPid) -> + gen_server:call(ClientPid, stop_session, infinity). + +stop(CPid) -> + gen_server:call(CPid, stop, infinity). + +get_last_message(Pid) -> + gen_server:call(Pid, get_last_message, infinity). + +init([ClientId]) -> + erlang:process_flag(trap_exit, true), + {ok, #state{clean_start = true, + client_id = ClientId, + last_msg = undefined + } + }. + +handle_call({start_session, ClientPid, ClientId, Attrs}, _From, State) -> + {ok, SessPid} = emqx_sm:open_session(Attrs), + {reply, {ok, SessPid}, + State#state{clean_start = true, + client_id = ClientId, + client_pid = ClientPid, + session_pid = SessPid + }}; +handle_call(stop_session, _From, #state{session_pid = Pid} = State) -> + is_pid(Pid) andalso is_process_alive(Pid) andalso emqx_sm:close_session(Pid), + {stop, normal, ok, State#state{session_pid = undefined}}; +handle_call(get_last_message, _From, #state{last_msg = Msg} = State) -> + {reply, Msg, State}; +handle_call(stop, _From, State) -> + {stop, normal, ok, State}; +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info({deliver, Msg}, State) -> + {noreply, State#state{last_msg = Msg}}; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +default_session_attributes(Zone, ClientId, ClientPid) -> + #{zone => Zone, + client_id => ClientId, + conn_pid => ClientPid, + clean_start => true, + username => undefined, + expiry_interval => 0, + max_inflight => 0, + topic_alias_maximum => 0, + will_msg => undefined + }. + diff --git a/test/emqx_mod_SUITE.erl b/test/emqx_mod_SUITE.erl new file mode 100644 index 000000000..44376f7b7 --- /dev/null +++ b/test/emqx_mod_SUITE.erl @@ -0,0 +1,25 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mod_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). + +all() -> [mod_subscription_rep]. + +mod_subscription_rep(_) -> ok. + diff --git a/test/emqx_mod_rewrite_tests.erl b/test/emqx_mod_rewrite_tests.erl new file mode 100644 index 000000000..6fea7ee71 --- /dev/null +++ b/test/emqx_mod_rewrite_tests.erl @@ -0,0 +1,63 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mod_rewrite_tests). + +-include_lib("emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). + + +rules() -> + Rawrules1 = "x/# ^x/y/(.+)$ z/y/$1", + Rawrules2 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2", + Rawrules = [Rawrules1, Rawrules2], + Rules = lists:map(fun(Rule) -> + [Topic, Re, Dest] = string:tokens(Rule, " "), + {rewrite, + list_to_binary(Topic), + list_to_binary(Re), + list_to_binary(Dest)} + end, Rawrules), + lists:map(fun({rewrite, Topic, Re, Dest}) -> + {ok, MP} = re:compile(Re), + {rewrite, Topic, MP, Dest} + end, Rules). + +rewrite_subscribe_test() -> + Rules = rules(), + io:format("Rules: ~p",[Rules]), + ?assertEqual({ok, [{<<"test">>, opts}]}, + emqx_mod_rewrite:rewrite_subscribe(credentials, [{<<"test">>, opts}], Rules)), + ?assertEqual({ok, [{<<"z/y/test">>, opts}]}, + emqx_mod_rewrite:rewrite_subscribe(credentials, [{<<"x/y/test">>, opts}], Rules)), + ?assertEqual({ok, [{<<"y/z/test_topic">>, opts}]}, + emqx_mod_rewrite:rewrite_subscribe(credentials, [{<<"y/test/z/test_topic">>, opts}], Rules)). + +rewrite_unsubscribe_test() -> + Rules = rules(), + ?assertEqual({ok, [{<<"test">>, opts}]}, + emqx_mod_rewrite:rewrite_subscribe(credentials, [{<<"test">>, opts}], Rules)), + ?assertEqual({ok, [{<<"z/y/test">>, opts}]}, + emqx_mod_rewrite:rewrite_subscribe(credentials, [{<<"x/y/test">>, opts}], Rules)), + ?assertEqual({ok, [{<<"y/z/test_topic">>, opts}]}, + emqx_mod_rewrite:rewrite_subscribe(credentials, [{<<"y/test/z/test_topic">>, opts}], Rules)). + +rewrite_publish_test() -> + Rules = rules(), + ?assertMatch({ok, #message{topic = <<"test">>}}, + emqx_mod_rewrite:rewrite_publish(#message{topic = <<"test">>}, Rules)), + ?assertMatch({ok, #message{topic = <<"z/y/test">>}}, + emqx_mod_rewrite:rewrite_publish(#message{topic = <<"x/y/test">>}, Rules)), + ?assertMatch({ok, #message{topic = <<"y/z/test_topic">>}}, + emqx_mod_rewrite:rewrite_publish(#message{topic = <<"y/test/z/test_topic">>}, Rules)). diff --git a/test/emqx_mod_sup_SUITE.erl b/test/emqx_mod_sup_SUITE.erl new file mode 100644 index 000000000..8169c3f91 --- /dev/null +++ b/test/emqx_mod_sup_SUITE.erl @@ -0,0 +1,28 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mod_sup_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). + +all() -> [t_child_all]. + +t_child_all(_) -> + {ok, _Pid} = emqx_mod_sup:start_link(), + {ok, _Child} = emqx_mod_sup:start_child(emqx_banned, worker), + timer:sleep(10), + ok = emqx_mod_sup:stop_child(emqx_banned). diff --git a/test/emqx_mountpoint_SUITE.erl b/test/emqx_mountpoint_SUITE.erl new file mode 100644 index 000000000..a77baf751 --- /dev/null +++ b/test/emqx_mountpoint_SUITE.erl @@ -0,0 +1,36 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mountpoint_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> [t_mount_unmount, t_replvar]. + +t_mount_unmount(_) -> + Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), + Msg2 = emqx_mountpoint:mount(<<"mount">>, Msg), + ?assertEqual(<<"mounttopic">>, Msg2#message.topic), + TopicFilter = [{<<"mounttopic">>, #{qos => ?QOS_2}}], + TopicFilter = emqx_mountpoint:mount(<<"mount">>, [{<<"topic">>, #{qos => ?QOS_2}}]), + Msg = emqx_mountpoint:unmount(<<"mount">>, Msg2). + +t_replvar(_) -> + <<"mount/test/clientid">> = emqx_mountpoint:replvar(<<"mount/%u/%c">>, #{client_id => <<"clientid">>, username => <<"test">>}). diff --git a/test/emqx_mqtt_caps_SUITE.erl b/test/emqx_mqtt_caps_SUITE.erl new file mode 100644 index 000000000..26e343ca6 --- /dev/null +++ b/test/emqx_mqtt_caps_SUITE.erl @@ -0,0 +1,131 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mqtt_caps_SUITE). + +-include_lib("eunit/include/eunit.hrl"). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +%% CT +-compile(export_all). +-compile(nowarn_export_all). + +all() -> [t_get_set_caps, t_check_pub, t_check_sub]. + +t_get_set_caps(_) -> + {ok, _} = emqx_zone:start_link(), + Caps = #{ + max_packet_size => ?MAX_PACKET_SIZE, + max_clientid_len => ?MAX_CLIENTID_LEN, + max_topic_alias => 0, + max_topic_levels => 0, + max_qos_allowed => ?QOS_2, + mqtt_retain_available => true, + mqtt_shared_subscription => true, + mqtt_wildcard_subscription => true + }, + Caps2 = Caps#{max_packet_size => 1048576}, + case emqx_mqtt_caps:get_caps(zone) of + Caps -> ok; + Caps2 -> ok + end, + PubCaps = #{ + max_qos_allowed => ?QOS_2, + mqtt_retain_available => true, + max_topic_alias => 0 + }, + PubCaps = emqx_mqtt_caps:get_caps(zone, publish), + NewPubCaps = PubCaps#{max_qos_allowed => ?QOS_1}, + emqx_zone:set_env(zone, '$mqtt_pub_caps', NewPubCaps), + timer:sleep(100), + NewPubCaps = emqx_mqtt_caps:get_caps(zone, publish), + SubCaps = #{ + max_topic_levels => 0, + max_qos_allowed => ?QOS_2, + mqtt_shared_subscription => true, + mqtt_wildcard_subscription => true + }, + SubCaps = emqx_mqtt_caps:get_caps(zone, subscribe), + emqx_zone:stop(). + +t_check_pub(_) -> + {ok, _} = emqx_zone:start_link(), + PubCaps = #{ + max_qos_allowed => ?QOS_1, + mqtt_retain_available => false, + max_topic_alias => 4 + }, + emqx_zone:set_env(zone, '$mqtt_pub_caps', PubCaps), + timer:sleep(100), + ct:log("~p", [emqx_mqtt_caps:get_caps(zone, publish)]), + BadPubProps1 = #{ + qos => ?QOS_2, + retain => false + }, + {error, ?RC_QOS_NOT_SUPPORTED} = emqx_mqtt_caps:check_pub(zone, BadPubProps1), + BadPubProps2 = #{ + qos => ?QOS_1, + retain => true + }, + {error, ?RC_RETAIN_NOT_SUPPORTED} = emqx_mqtt_caps:check_pub(zone, BadPubProps2), + BadPubProps3 = #{ + qos => ?QOS_1, + retain => false, + topic_alias => 5 + }, + {error, ?RC_TOPIC_ALIAS_INVALID} = emqx_mqtt_caps:check_pub(zone, BadPubProps3), + PubProps = #{ + qos => ?QOS_1, + retain => false + }, + ok = emqx_mqtt_caps:check_pub(zone, PubProps), + emqx_zone:stop(). + +t_check_sub(_) -> + {ok, _} = emqx_zone:start_link(), + + Opts = #{qos => ?QOS_2, share => true, rc => 0}, + Caps = #{ + max_topic_levels => 0, + max_qos_allowed => ?QOS_2, + mqtt_shared_subscription => true, + mqtt_wildcard_subscription => true + }, + + ok = do_check_sub([{<<"client/stat">>, Opts}], [{<<"client/stat">>, Opts}]), + ok = do_check_sub(Caps#{max_qos_allowed => ?QOS_1}, [{<<"client/stat">>, Opts}], [{<<"client/stat">>, Opts#{qos => ?QOS_1}}]), + ok = do_check_sub(Caps#{max_topic_levels => 1}, + [{<<"client/stat">>, Opts}], + [{<<"client/stat">>, Opts#{rc => ?RC_TOPIC_FILTER_INVALID}}]), + ok = do_check_sub(Caps#{mqtt_shared_subscription => false}, + [{<<"client/stat">>, Opts}], + [{<<"client/stat">>, Opts#{rc => ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}}]), + + ok = do_check_sub(Caps#{mqtt_wildcard_subscription => false}, + [{<<"vlient/+/dsofi">>, Opts}], + [{<<"vlient/+/dsofi">>, Opts#{rc => ?RC_WILDCARD_SUBSCRIPTIONS_NOT_SUPPORTED}}]), + emqx_zone:stop(). + + + +do_check_sub(TopicFilters, Topics) -> + {ok, Topics} = emqx_mqtt_caps:check_sub(zone, TopicFilters), + ok. +do_check_sub(Caps, TopicFilters, Topics) -> + emqx_zone:set_env(zone, '$mqtt_sub_caps', Caps), + timer:sleep(100), + {_, Topics} = emqx_mqtt_caps:check_sub(zone, TopicFilters), + ok. diff --git a/test/emqx_mqtt_packet_SUITE.erl b/test/emqx_mqtt_packet_SUITE.erl new file mode 100644 index 000000000..4386ff02e --- /dev/null +++ b/test/emqx_mqtt_packet_SUITE.erl @@ -0,0 +1,117 @@ +%%%=================================================================== +%%% Copyright (c) 2013-2018 EMQ Inc. All rights reserved. +%%% +%%% Licensed under the Apache License, Version 2.0 (the "License"); +%%% you may not use this file except in compliance with the License. +%%% You may obtain a copy of the License at +%%% +%%% http://www.apache.org/licenses/LICENSE-2.0 +%%% +%%% Unless required by applicable law or agreed to in writing, software +%%% distributed under the License is distributed on an "AS IS" BASIS, +%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%%% See the License for the specific language governing permissions and +%%% limitations under the License. +%%%=================================================================== + +-module(emqx_mqtt_packet_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-import(emqx_frame, [serialize/1]). + +-include("emqx_mqtt.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +-define(INVALID_RESERVED, 1). + +-define(CONNECT_INVALID_PACKET(Var), + #mqtt_packet{header = #mqtt_packet_header{type = ?INVALID_RESERVED}, + variable = Var}). + +-define(CASE1_PROTOCOL_NAME, ?CONNECT_PACKET(#mqtt_packet_connect{ + proto_name = <<"MQTC">>, + client_id = <<"mqtt_protocol_name">>, + username = <<"admin">>, + password = <<"public">>})). + +-define(CASE2_PROTOCAL_VER, ?CONNECT_PACKET(#mqtt_packet_connect{ + client_id = <<"mqtt_client">>, + proto_ver = 6, + username = <<"admin">>, + password = <<"public">>})). + +-define(CASE3_PROTOCAL_INVALID_RESERVED, ?CONNECT_INVALID_PACKET(#mqtt_packet_connect{ + client_id = <<"mqtt_client">>, + proto_ver = 5, + username = <<"admin">>, + password = <<"public">>})). + +-define(PROTOCOL5, ?CONNECT_PACKET(#mqtt_packet_connect{ + proto_ver = 5, + keepalive = 60, + properties = #{'Message-Expiry-Interval' => 3600}, + client_id = <<"mqtt_client">>, + will_topic = <<"will_tipic">>, + will_payload = <<"will message">>, + username = <<"admin">>, + password = <<"public">>})). + + + +all() -> [{group, connect}]. + +groups() -> [{connect, [sequence], + [case1_protocol_name, + case2_protocol_ver%, + %TOTO case3_invalid_reserved + ]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +case1_protocol_name(_) -> + {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), + MqttPacket = serialize(?CASE1_PROTOCOL_NAME), + emqx_client_sock:send(Sock, MqttPacket), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?CONNACK_PROTO_VER), _} = raw_recv_pase(Data), + Disconnect = gen_tcp:recv(Sock, 0), + ?assertEqual({error, closed}, Disconnect). + +case2_protocol_ver(_) -> + {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), + Packet = serialize(?CASE2_PROTOCAL_VER), + emqx_client_sock:send(Sock, Packet), + {ok, Data} = gen_tcp:recv(Sock, 0), + %% case1 Unacceptable protocol version + {ok, ?CONNACK_PACKET(?CONNACK_PROTO_VER), _} = raw_recv_pase(Data), + Disconnect = gen_tcp:recv(Sock, 0), + ?assertEqual({error, closed}, Disconnect). + +case3_invalid_reserved(_) -> + {ok, Sock} = emqx_client_sock:connect({127,0,0,1}, 1883, [binary, {packet, raw}, {active, false}], 3000), + Packet = serialize(?CASE3_PROTOCAL_INVALID_RESERVED), + emqx_client_sock:send(Sock, Packet), + {ok, Data} = gen_tcp:recv(Sock, 0), + %% case1 Unacceptable protocol version + ct:log("Data:~p~n", [raw_recv_pase(Data)]), + {ok, ?CONNACK_PACKET(?CONNACK_PROTO_VER), _} = raw_recv_pase(Data), + Disconnect = gen_tcp:recv(Sock, 0), + ?assertEqual({error, closed}, Disconnect). + +raw_recv_pase(P) -> + emqx_frame:parse(P, {none, #{max_packet_size => ?MAX_PACKET_SIZE, + version => ?MQTT_PROTO_V4} }). diff --git a/test/emqx_mqtt_props_SUITE.erl b/test/emqx_mqtt_props_SUITE.erl new file mode 100644 index 000000000..8d3b16a14 --- /dev/null +++ b/test/emqx_mqtt_props_SUITE.erl @@ -0,0 +1,28 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mqtt_props_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_mqtt.hrl"). + +all() -> [t_mqtt_properties_all]. + +t_mqtt_properties_all(_) -> + Props = emqx_mqtt_props:filter(?CONNECT, #{'Session-Expiry-Interval' => 1, 'Maximum-Packet-Size' => 255}), + ok = emqx_mqtt_props:validate(Props), + #{} = emqx_mqtt_props:filter(?CONNECT, #{'Maximum-QoS' => ?QOS_2}). + diff --git a/test/emqx_mqueue_SUITE.erl b/test/emqx_mqueue_SUITE.erl new file mode 100644 index 000000000..9bb424fa1 --- /dev/null +++ b/test/emqx_mqueue_SUITE.erl @@ -0,0 +1,142 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_mqueue_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). +-include("emqx_mqtt.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +-define(Q, emqx_mqueue). + +all() -> [t_in, t_in_qos0, t_out, t_simple_mqueue, t_infinity_simple_mqueue, + t_priority_mqueue, t_infinity_priority_mqueue]. + +t_in(_) -> + Opts = #{max_len => 5, store_qos0 => true}, + Q = ?Q:init(Opts), + ?assert(?Q:is_empty(Q)), + {_, Q1} = ?Q:in(#message{}, Q), + ?assertEqual(1, ?Q:len(Q1)), + {_, Q2} = ?Q:in(#message{qos = 1}, Q1), + ?assertEqual(2, ?Q:len(Q2)), + {_, Q3} = ?Q:in(#message{qos = 2}, Q2), + {_, Q4} = ?Q:in(#message{}, Q3), + {_, Q5} = ?Q:in(#message{}, Q4), + ?assertEqual(5, ?Q:len(Q5)). + +t_in_qos0(_) -> + Opts = #{max_len => 5, store_qos0 => false}, + Q = ?Q:init(Opts), + {_, Q1} = ?Q:in(#message{qos = 0}, Q), + ?assert(?Q:is_empty(Q1)), + {_, Q2} = ?Q:in(#message{qos = 0}, Q1), + ?assert(?Q:is_empty(Q2)). + +t_out(_) -> + Opts = #{max_len => 5, store_qos0 => true}, + Q = ?Q:init(Opts), + {empty, Q} = ?Q:out(Q), + {_, Q1} = ?Q:in(#message{}, Q), + {Value, Q2} = ?Q:out(Q1), + ?assertEqual(0, ?Q:len(Q2)), + ?assertEqual({value, #message{}}, Value). + +t_simple_mqueue(_) -> + Opts = #{max_len => 3, store_qos0 => false}, + Q = ?Q:init(Opts), + ?assertEqual(3, ?Q:max_len(Q)), + ?assert(?Q:is_empty(Q)), + {_, Q1} = ?Q:in(#message{qos = 1, payload = <<"1">>}, Q), + {_, Q2} = ?Q:in(#message{qos = 1, payload = <<"2">>}, Q1), + {_, Q3} = ?Q:in(#message{qos = 1, payload = <<"3">>}, Q2), + {_, Q4} = ?Q:in(#message{qos = 1, payload = <<"4">>}, Q3), + ?assertEqual(3, ?Q:len(Q4)), + {{value, Msg}, Q5} = ?Q:out(Q4), + ?assertEqual(<<"2">>, Msg#message.payload), + ?assertEqual([{len, 2}, {max_len, 3}, {dropped, 1}], ?Q:stats(Q5)). + +t_infinity_simple_mqueue(_) -> + Opts = #{max_len => 0, store_qos0 => false}, + Q = ?Q:init(Opts), + ?assert(?Q:is_empty(Q)), + ?assertEqual(0, ?Q:max_len(Q)), + Qx = lists:foldl( + fun(I, AccQ) -> + {_, NewQ} = ?Q:in(#message{qos = 1, payload = iolist_to_binary([I])}, AccQ), + NewQ + end, Q, lists:seq(1, 255)), + ?assertEqual(255, ?Q:len(Qx)), + ?assertEqual([{len, 255}, {max_len, 0}, {dropped, 0}], ?Q:stats(Qx)), + {{value, V}, _Qy} = ?Q:out(Qx), + ?assertEqual(<<1>>, V#message.payload). + +t_priority_mqueue(_) -> + Opts = #{max_len => 3, + priorities => + #{<<"t1">> => 1, + <<"t2">> => 2, + <<"t3">> => 3 + }, + store_qos0 => false}, + Q = ?Q:init(Opts), + ?assertEqual(3, ?Q:max_len(Q)), + ?assert(?Q:is_empty(Q)), + {_, Q1} = ?Q:in(#message{qos = 1, topic = <<"t2">>}, Q), + {_, Q2} = ?Q:in(#message{qos = 1, topic = <<"t1">>}, Q1), + {_, Q3} = ?Q:in(#message{qos = 1, topic = <<"t3">>}, Q2), + ?assertEqual(3, ?Q:len(Q3)), + {_, Q4} = ?Q:in(#message{qos = 1, topic = <<"t2">>}, Q3), + ?assertEqual(4, ?Q:len(Q4)), + {_, Q5} = ?Q:in(#message{qos = 1, topic = <<"t2">>}, Q4), + ?assertEqual(5, ?Q:len(Q5)), + {_, Q6} = ?Q:in(#message{qos = 1, topic = <<"t2">>}, Q5), + ?assertEqual(5, ?Q:len(Q6)), + {{value, Msg}, Q7} = ?Q:out(Q6), + ?assertEqual(4, ?Q:len(Q7)), + ?assertEqual(<<"t3">>, Msg#message.topic). + +t_infinity_priority_mqueue(_) -> + Opts = #{max_len => 0, + priorities => + #{<<"t">> => 1, + <<"t1">> => 2 + }, + store_qos0 => false}, + Q = ?Q:init(Opts), + ?assertEqual(0, ?Q:max_len(Q)), + Qx = lists:foldl(fun(I, AccQ) -> + {undefined, AccQ1} = ?Q:in(#message{topic = <<"t1">>, qos = 1, payload = iolist_to_binary([I])}, AccQ), + {undefined, AccQ2} = ?Q:in(#message{topic = <<"t">>, qos = 1, payload = iolist_to_binary([I])}, AccQ1), + AccQ2 + end, Q, lists:seq(1, 255)), + ?assertEqual(510, ?Q:len(Qx)), + ?assertEqual([{len, 510}, {max_len, 0}, {dropped, 0}], ?Q:stats(Qx)). + +t_priority_mqueue2(_) -> + Opts = #{max_length => 2, store_qos0 => false}, + Q = ?Q:init("priority_queue2_test", Opts), + 2 = ?Q:max_len(Q), + {_, Q1} = ?Q:in(#message{topic = <<"x">>, qos = 1, payload = <<1>>}, Q), + {_, Q2} = ?Q:in(#message{topic = <<"x">>, qos = 1, payload = <<2>>}, Q1), + {_, Q3} = ?Q:in(#message{topic = <<"y">>, qos = 1, payload = <<3>>}, Q2), + {_, Q4} = ?Q:in(#message{topic = <<"y">>, qos = 1, payload = <<4>>}, Q3), + 4 = ?Q:len(Q4), + {{value, _Val}, Q5} = ?Q:out(Q4), + 3 = ?Q:len(Q5). + diff --git a/test/emqttd_net_SUITE.erl b/test/emqx_net_SUITE.erl similarity index 75% rename from test/emqttd_net_SUITE.erl rename to test/emqx_net_SUITE.erl index c6bb10c14..50a830d10 100644 --- a/test/emqttd_net_SUITE.erl +++ b/test/emqx_net_SUITE.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,12 +11,12 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_net_SUITE). +-module(emqx_net_SUITE). %% CT -compile(export_all). +-compile(nowarn_export_all). all() -> [{group, keepalive}]. @@ -28,13 +27,13 @@ groups() -> [{keepalive, [], [t_keepalive]}]. %%-------------------------------------------------------------------- t_keepalive(_) -> - {ok, KA} = emqttd_keepalive:start(fun() -> {ok, 1} end, 1, {keepalive, timeout}), + {ok, KA} = emqx_keepalive:start(fun() -> {ok, 1} end, 1, {keepalive, timeout}), [resumed, timeout] = lists:reverse(keepalive_recv(KA, [])). keepalive_recv(KA, Acc) -> receive {keepalive, timeout} -> - case emqttd_keepalive:check(KA) of + case emqx_keepalive:check(KA) of {ok, KA1} -> keepalive_recv(KA1, [resumed | Acc]); {error, timeout} -> [timeout | Acc] end diff --git a/test/emqx_packet_SUITE.erl b/test/emqx_packet_SUITE.erl new file mode 100644 index 000000000..42fcbc9d7 --- /dev/null +++ b/test/emqx_packet_SUITE.erl @@ -0,0 +1,131 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + + +-module(emqx_packet_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). + +-include("emqx_mqtt.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [ + packet_proto_name, + packet_type_name, + packet_validate, + packet_message, + packet_format, + packet_will_msg + ]. + +packet_proto_name(_) -> + ?assertEqual(<<"MQIsdp">>, emqx_packet:protocol_name(3)), + ?assertEqual(<<"MQTT">>, emqx_packet:protocol_name(4)), + ?assertEqual(<<"MQTT">>, emqx_packet:protocol_name(5)). + +packet_type_name(_) -> + ?assertEqual('CONNECT', emqx_packet:type_name(?CONNECT)), + ?assertEqual('UNSUBSCRIBE', emqx_packet:type_name(?UNSUBSCRIBE)). + +packet_validate(_) -> + ?assert(emqx_packet:validate(?SUBSCRIBE_PACKET(15, #{'Subscription-Identifier' => 1}, [{<<"topic">>, #{qos => ?QOS_0}}]))), + ?assert(emqx_packet:validate(?UNSUBSCRIBE_PACKET(89, [<<"topic">>]))), + ?assert(emqx_packet:validate(?CONNECT_PACKET(#mqtt_packet_connect{}))), + ?assert(emqx_packet:validate(?PUBLISH_PACKET(1, <<"topic">>, 1, #{'Response-Topic' => <<"responsetopic">>, 'Topic-Alias' => 1}, <<"payload">>))), + ?assert(emqx_packet:validate(?CONNECT_PACKET(#mqtt_packet_connect{properties = #{'Receive-Maximum' => 1}}))), + ?assertError(subscription_identifier_invalid, + emqx_packet:validate( + ?SUBSCRIBE_PACKET(15, #{'Subscription-Identifier' => -1}, + [{<<"topic">>, #{qos => ?QOS_0}}]))), + ?assertError(topic_filters_invalid, + emqx_packet:validate(?UNSUBSCRIBE_PACKET(1,[]))), + ?assertError(topic_name_invalid, + emqx_packet:validate(?PUBLISH_PACKET(1,<<>>,1,#{},<<"payload">>))), + ?assertError(topic_name_invalid, + emqx_packet:validate(?PUBLISH_PACKET + (1, <<"+/+">>, 1, #{}, <<"payload">>))), + ?assertError(topic_alias_invalid, + emqx_packet:validate( + ?PUBLISH_PACKET + (1, <<"topic">>, 1, #{'Topic-Alias' => 0}, <<"payload">>))), + ?assertError(protocol_error, + emqx_packet:validate( + ?PUBLISH_PACKET(1, <<"topic">>, 1, + #{'Subscription-Identifier' => 10}, <<"payload">>))), + ?assertError(protocol_error, + emqx_packet:validate( + ?PUBLISH_PACKET(1, <<"topic">>, 1, + #{'Response-Topic' => <<"+/+">>}, <<"payload">>))), + ?assertError(protocol_error, + emqx_packet:validate( + ?CONNECT_PACKET(#mqtt_packet_connect{ + properties = + #{'Request-Response-Information' => -1}}))), + ?assertError(protocol_error, + emqx_packet:validate( + ?CONNECT_PACKET(#mqtt_packet_connect{ + properties = + #{'Request-Problem-Information' => 2}}))), + ?assertError(protocol_error, + emqx_packet:validate( + ?CONNECT_PACKET(#mqtt_packet_connect{ + properties = + #{'Receive-Maximum' => 0}}))). + +packet_message(_) -> + Pkt = #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, + qos = ?QOS_0, + retain = false, + dup = false}, + variable = #mqtt_packet_publish{topic_name = <<"topic">>, + packet_id = 10, + properties = #{}}, + payload = <<"payload">>}, + Msg = emqx_message:make(<<"clientid">>, ?QOS_0, <<"topic">>, <<"payload">>), + Msg2 = emqx_message:set_flag(retain, false, Msg), + Pkt = emqx_packet:from_message(10, Msg2), + Msg3 = emqx_message:set_header(username, "test", Msg2), + Msg4 = emqx_packet:to_message(#{client_id => <<"clientid">>, username => "test"}, Pkt), + Msg5 = Msg4#message{timestamp = Msg3#message.timestamp}, + Msg5 = Msg3. + +packet_format(_) -> + io:format("~s", [emqx_packet:format(?CONNECT_PACKET(#mqtt_packet_connect{}))]), + io:format("~s", [emqx_packet:format(?CONNACK_PACKET(?CONNACK_SERVER))]), + io:format("~s", [emqx_packet:format(?PUBLISH_PACKET(?QOS_1, 1))]), + io:format("~s", [emqx_packet:format(?PUBLISH_PACKET(?QOS_2, <<"topic">>, 10, <<"payload">>))]), + io:format("~s", [emqx_packet:format(?PUBACK_PACKET(?PUBACK, 98))]), + io:format("~s", [emqx_packet:format(?PUBREL_PACKET(99))]), + io:format("~s", [emqx_packet:format(?SUBSCRIBE_PACKET(15, [{<<"topic">>, ?QOS_0}, {<<"topic1">>, ?QOS_1}]))]), + io:format("~s", [emqx_packet:format(?SUBACK_PACKET(40, [?QOS_0, ?QOS_1]))]), + io:format("~s", [emqx_packet:format(?UNSUBSCRIBE_PACKET(89, [<<"t">>, <<"t2">>]))]), + io:format("~s", [emqx_packet:format(?UNSUBACK_PACKET(90))]). + +packet_will_msg(_) -> + Pkt = #mqtt_packet_connect{ will_flag = true, + client_id = <<"clientid">>, + username = "test", + will_retain = true, + will_qos = ?QOS_2, + will_topic = <<"topic">>, + will_props = #{}, + will_payload = <<"payload">>}, + Msg = emqx_packet:will_msg(Pkt), + ?assertEqual(<<"clientid">>, Msg#message.from), + ?assertEqual(<<"topic">>, Msg#message.topic). diff --git a/test/emqx_pd_SUITE.erl b/test/emqx_pd_SUITE.erl new file mode 100644 index 000000000..e53fa7539 --- /dev/null +++ b/test/emqx_pd_SUITE.erl @@ -0,0 +1,31 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pd_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> [update_counter]. + +update_counter(_) -> + ?assertEqual(undefined, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(1, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(2, emqx_pd:update_counter(bytes, 1)), + ?assertEqual(3, emqx_pd:get_counter(bytes)), + ?assertEqual(3, emqx_pd:reset_counter(bytes)), + ?assertEqual(0, emqx_pd:get_counter(bytes)). + diff --git a/test/emqx_pmon_SUITE.erl b/test/emqx_pmon_SUITE.erl new file mode 100644 index 000000000..67e8cf4d7 --- /dev/null +++ b/test/emqx_pmon_SUITE.erl @@ -0,0 +1,48 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pmon_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [t_monitor, t_find, t_erase]. + +t_monitor(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), PMon), + ?assertEqual(1, emqx_pmon:count(PMon1)), + PMon2 = emqx_pmon:demonitor(self(), PMon1), + ?assertEqual(0, emqx_pmon:count(PMon2)). + +t_find(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), val, PMon), + ?assertEqual(1, emqx_pmon:count(PMon1)), + ?assertEqual({ok, val}, emqx_pmon:find(self(), PMon1)), + PMon2 = emqx_pmon:erase(self(), PMon1), + ?assertEqual(error, emqx_pmon:find(self(), PMon2)). + +t_erase(_) -> + PMon = emqx_pmon:new(), + PMon1 = emqx_pmon:monitor(self(), val, PMon), + PMon2 = emqx_pmon:erase(self(), PMon1), + ?assertEqual(0, emqx_pmon:count(PMon2)), + {Items, PMon3} = emqx_pmon:erase_all([self()], PMon1), + ?assertEqual([{self(), val}], Items), + ?assertEqual(0, emqx_pmon:count(PMon3)). + diff --git a/test/emqx_pool_SUITE.erl b/test/emqx_pool_SUITE.erl new file mode 100644 index 000000000..3d7d0f7e5 --- /dev/null +++ b/test/emqx_pool_SUITE.erl @@ -0,0 +1,65 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pool_SUITE). + +-compile(export_all). + +-compile(nowarn_export_all). + +-include("emqx_mqtt.hrl"). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> [ + {group, submit_case}, + {group, async_submit_case} + ]. + +groups() -> + [ + {submit_case, [sequence], [submit_mfa, submit_fa]}, + {async_submit_case, [sequence], [async_submit_mfa]} + ]. + +init_per_suite(Config) -> + application:ensure_all_started(gproc), + Config. + +end_per_suite(_Config) -> + ok. + +submit_mfa(_Config) -> + erlang:process_flag(trap_exit, true), + {ok, Pid} = emqx_pool:start_link(), + Result = emqx_pool:submit({?MODULE, test_mfa, []}), + ?assertEqual(15, Result), + gen_server:stop(Pid, normal, 3000), + ok. + +submit_fa(_Config) -> + {ok, Pid} = emqx_pool:start_link(), + Fun = fun(X) -> case X rem 2 of 0 -> {true, X div 2}; _ -> false end end, + Result = emqx_pool:submit(Fun, [2]), + ?assertEqual({true, 1}, Result), + exit(Pid, normal). + +test_mfa() -> + lists:foldl(fun(X, Sum) -> X + Sum end, 0, [1,2,3,4,5]). + +async_submit_mfa(_Config) -> + {ok, Pid} = emqx_pool:start_link(), + emqx_pool:async_submit({?MODULE, test_mfa, []}), + exit(Pid, normal). + diff --git a/test/emqx_pqueue_SUITE.erl b/test/emqx_pqueue_SUITE.erl new file mode 100644 index 000000000..9efccf472 --- /dev/null +++ b/test/emqx_pqueue_SUITE.erl @@ -0,0 +1,123 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_pqueue_SUITE). + +-include("emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(PQ, emqx_pqueue). + +all() -> [t_priority_queue_plen, t_priority_queue_out2, t_priority_queues]. + +t_priority_queue_plen(_) -> + Q = ?PQ:new(), + 0 = ?PQ:plen(0, Q), + Q0 = ?PQ:in(z, Q), + 1 = ?PQ:plen(0, Q0), + Q1 = ?PQ:in(x, 1, Q0), + 1 = ?PQ:plen(1, Q1), + Q2 = ?PQ:in(y, 2, Q1), + 1 = ?PQ:plen(2, Q2), + Q3 = ?PQ:in(z, 2, Q2), + 2 = ?PQ:plen(2, Q3), + {_, Q4} = ?PQ:out(1, Q3), + 0 = ?PQ:plen(1, Q4), + {_, Q5} = ?PQ:out(Q4), + 1 = ?PQ:plen(2, Q5), + {_, Q6} = ?PQ:out(Q5), + 0 = ?PQ:plen(2, Q6), + 1 = ?PQ:len(Q6), + {_, Q7} = ?PQ:out(Q6), + 0 = ?PQ:len(Q7). + +t_priority_queue_out2(_) -> + Els = [a, {b, 1}, {c, 1}, {d, 2}, {e, 2}, {f, 2}], + Q = ?PQ:new(), + Q0 = lists:foldl( + fun({El, P}, Acc) -> + ?PQ:in(El, P, Acc); + (El, Acc) -> + ?PQ:in(El, Acc) + end, Q, Els), + {Val, Q1} = ?PQ:out(Q0), + {value, d} = Val, + {Val1, Q2} = ?PQ:out(2, Q1), + {value, e} = Val1, + {Val2, Q3} = ?PQ:out(1, Q2), + {value, b} = Val2, + {Val3, Q4} = ?PQ:out(Q3), + {value, f} = Val3, + {Val4, Q5} = ?PQ:out(Q4), + {value, c} = Val4, + {Val5, Q6} = ?PQ:out(Q5), + {value, a} = Val5, + {empty, _Q7} = ?PQ:out(Q6). + +t_priority_queues(_) -> + Q0 = ?PQ:new(), + Q1 = ?PQ:new(), + PQueue = {pqueue, [{0, Q0}, {1, Q1}]}, + ?assert(?PQ:is_queue(PQueue)), + [] = ?PQ:to_list(PQueue), + + PQueue1 = ?PQ:in(a, 0, ?PQ:new()), + PQueue2 = ?PQ:in(b, 0, PQueue1), + + PQueue3 = ?PQ:in(c, 1, PQueue2), + PQueue4 = ?PQ:in(d, 1, PQueue3), + + 4 = ?PQ:len(PQueue4), + + [{1, c}, {1, d}, {0, a}, {0, b}] = ?PQ:to_list(PQueue4), + PQueue4 = ?PQ:from_list([{1, c}, {1, d}, {0, a}, {0, b}]), + + empty = ?PQ:highest(?PQ:new()), + 0 = ?PQ:highest(PQueue1), + 1 = ?PQ:highest(PQueue4), + + PQueue5 = ?PQ:in(e, infinity, PQueue4), + PQueue6 = ?PQ:in(f, 1, PQueue5), + + {{value, e}, PQueue7} = ?PQ:out(PQueue6), + {empty, _} = ?PQ:out(0, ?PQ:new()), + + {empty, Q0} = ?PQ:out_p(Q0), + + Q2 = ?PQ:in(a, Q0), + Q3 = ?PQ:in(b, Q2), + Q4 = ?PQ:in(c, Q3), + + {{value, a, 0}, _Q5} = ?PQ:out_p(Q4), + + {{value,c,1}, PQueue8} = ?PQ:out_p(PQueue7), + + Q4 = ?PQ:join(Q4, ?PQ:new()), + Q4 = ?PQ:join(?PQ:new(), Q4), + + {queue, [a], [a], 2} = ?PQ:join(Q2, Q2), + + {pqueue,[{-1,{queue,[f],[d],2}}, + {0,{queue,[a],[a,b],3}}]} = ?PQ:join(PQueue8, Q2), + + {pqueue,[{-1,{queue,[f],[d],2}}, + {0,{queue,[b],[a,a],3}}]} = ?PQ:join(Q2, PQueue8), + + {pqueue,[{-1,{queue,[f],[d,f,d],4}}, + {0,{queue,[b],[a,b,a],4}}]} = ?PQ:join(PQueue8, PQueue8). + + diff --git a/test/emqx_protocol_SUITE.erl b/test/emqx_protocol_SUITE.erl new file mode 100644 index 000000000..3bf9976c7 --- /dev/null +++ b/test/emqx_protocol_SUITE.erl @@ -0,0 +1,604 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_protocol_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). + +-include("emqx_mqtt.hrl"). + +-define(TOPICS, [<<"TopicA">>, <<"TopicA/B">>, <<"Topic/C">>, <<"TopicA/C">>, + <<"/TopicA">>]). + +-define(CLIENT2, ?CONNECT_PACKET(#mqtt_packet_connect{ + username = <<"admin">>, + clean_start = false, + password = <<"public">>})). + +all() -> + [ + {group, mqtt_common}, + {group, mqttv4}, + {group, mqttv5}, + {group, acl} + ]. + +groups() -> + [{mqtt_common, + [sequence], + [will_check]}, + {mqttv4, + [sequence], + [connect_v4, + subscribe_v4]}, + {mqttv5, + [sequence], + [connect_v5, + subscribe_v5]}, + {acl, + [sequence], + [acl_deny_action]}]. + +init_per_suite(Config) -> + [start_apps(App, SchemaFile, ConfigFile) || + {App, SchemaFile, ConfigFile} + <- [{emqx, deps_path(emqx, "priv/emqx.schema"), + deps_path(emqx, "etc/emqx.conf")}]], + emqx_zone:set_env(external, max_topic_alias, 20), + Config. + +end_per_suite(_Config) -> + application:stop(emqx). + +batch_connect(NumberOfConnections) -> + batch_connect([], NumberOfConnections). + +batch_connect(Socks, 0) -> + Socks; +batch_connect(Socks, NumberOfConnections) -> + {ok, Sock} = emqx_client_sock:connect({127, 0, 0, 1}, 1883, + [binary, {packet, raw}, {active, false}], + 3000), + batch_connect([Sock | Socks], NumberOfConnections - 1). + +with_connection(DoFun, NumberOfConnections) -> + Socks = batch_connect(NumberOfConnections), + try + DoFun(Socks) + after + lists:foreach(fun(Sock) -> + emqx_client_sock:close(Sock) + end, Socks) + end. + +with_connection(DoFun) -> + with_connection(DoFun, 1). + + % {ok, Sock} = emqx_client_sock:connect({127, 0, 0, 1}, 1883, + % [binary, {packet, raw}, + % {active, false}], 3000), + % try + % DoFun(Sock) + % after + % emqx_client_sock:close(Sock) + % end. + +connect_v4(_) -> + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, raw_send_serialize(?PACKET(?PUBLISH))), + {error, closed} =gen_tcp:recv(Sock, 0) + end), + with_connection(fun([Sock]) -> + ConnectPacket = raw_send_serialize(?CONNECT_PACKET + (#mqtt_packet_connect{ + client_id = <<"mqttv4_client">>, + username = <<"admin">>, + password = <<"public">>, + proto_ver = ?MQTT_PROTO_V4 + })), + emqx_client_sock:send(Sock, ConnectPacket), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?CONNACK_ACCEPT), _} = raw_recv_parse(Data, ?MQTT_PROTO_V4), + + emqx_client_sock:send(Sock, ConnectPacket), + {error, closed} = gen_tcp:recv(Sock, 0) + end), + ok. + + +connect_v5(_) -> + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET(#mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Request-Response-Information' => -1}}))), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_PROTOCOL_ERROR), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5) + end), + + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Request-Problem-Information' => 2}}))), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_PROTOCOL_ERROR), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5) + end), + + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Request-Response-Information' => 1}}) + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0, + #{'Response-Information' := _RespInfo}), _} = + raw_recv_parse(Data, ?MQTT_PROTO_V5) + end), + + % topic alias = 0 + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Topic-Alias-Maximum' => 10}}), + #{version => ?MQTT_PROTO_V5} + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0, + #{'Topic-Alias-Maximum' := 20}), _} = + raw_recv_parse(Data, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 1, #{'Topic-Alias' => 0}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data2} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_TOPIC_ALIAS_INVALID), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5) + end), + + % topic alias maximum + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = + #{'Topic-Alias-Maximum' => 10}}), + #{version => ?MQTT_PROTO_V5} + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0, + #{'Topic-Alias-Maximum' := 20}), _} = + raw_recv_parse(Data, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, raw_send_serialize(?SUBSCRIBE_PACKET(1, [{<<"TopicA">>, #{rh => 1, + qos => ?QOS_2, + rap => 0, + nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5})), + + {ok, Data2} = gen_tcp:recv(Sock, 0), + {ok, ?SUBACK_PACKET(1, #{}, [2]), _} = raw_recv_parse(Data2, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 1, #{'Topic-Alias' => 15}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data3} = gen_tcp:recv(Sock, 0), + + {ok, ?PUBACK_PACKET(1, 0), _} = raw_recv_parse(Data3, ?MQTT_PROTO_V5), + + {ok, Data4} = gen_tcp:recv(Sock, 0), + + {ok, ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, _, <<"hello">>), _} = raw_recv_parse(Data4, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, + raw_send_serialize( + ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, 2, #{'Topic-Alias' => 21}, <<"hello">>), + #{version => ?MQTT_PROTO_V5} + )), + + {ok, Data5} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_TOPIC_ALIAS_INVALID), _} = raw_recv_parse(Data5, ?MQTT_PROTO_V5) + end), + + % test clean start + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + clean_start = true, + client_id = <<"myclient">>, + properties = + #{'Session-Expiry-Interval' => 10}}) + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5), + emqx_client_sock:send(Sock, raw_send_serialize( + ?DISCONNECT_PACKET(?RC_SUCCESS) + )) + end), + + timer:sleep(1000), + + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + clean_start = false, + client_id = <<"myclient">>, + properties = + #{'Session-Expiry-Interval' => 10}}) + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 1), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5) + end), + + % test will message publish and cancel + with_connection(fun([Sock]) -> + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + clean_start = true, + client_id = <<"myclient">>, + will_flag = true, + will_qos = ?QOS_1, + will_retain = false, + will_props = #{'Will-Delay-Interval' => 5}, + will_topic = <<"TopicA">>, + will_payload = <<"will message">>, + properties = #{'Session-Expiry-Interval' => 3} + } + ) + ) + ), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5), + + {ok, Sock2} = emqx_client_sock:connect({127, 0, 0, 1}, 1883, + [binary, {packet, raw}, + {active, false}], 3000), + + do_connect(Sock2, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock2, raw_send_serialize(?SUBSCRIBE_PACKET(1, [{<<"TopicA">>, #{rh => 1, + qos => ?QOS_2, + rap => 0, + nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5})), + + {ok, SubData} = gen_tcp:recv(Sock2, 0), + {ok, ?SUBACK_PACKET(1, #{}, [2]), _} = raw_recv_parse(SubData, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, raw_send_serialize( + ?DISCONNECT_PACKET(?RC_DISCONNECT_WITH_WILL_MESSAGE) + ) + ), + + {error, timeout} = gen_tcp:recv(Sock2, 0, 1000), + + % session resumed + {ok, Sock3} = emqx_client_sock:connect({127, 0, 0, 1}, 1883, + [binary, {packet, raw}, + {active, false}], 3000), + + emqx_client_sock:send(Sock3, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + clean_start = false, + client_id = <<"myclient">>, + will_flag = true, + will_qos = ?QOS_1, + will_retain = false, + will_props = #{'Will-Delay-Interval' => 5}, + will_topic = <<"TopicA">>, + will_payload = <<"will message 2">>, + properties = #{'Session-Expiry-Interval' => 3} + } + ) + ) + ), + {ok, Data3} = gen_tcp:recv(Sock3, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 1), _} = raw_recv_parse(Data3, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock3, raw_send_serialize( + ?DISCONNECT_PACKET(?RC_DISCONNECT_WITH_WILL_MESSAGE) + ) + ), + + {ok, WillData} = gen_tcp:recv(Sock2, 0), + {ok, ?PUBLISH_PACKET(?QOS_1, <<"TopicA">>, _, <<"will message 2">>), _} = raw_recv_parse(WillData, ?MQTT_PROTO_V5), + + emqx_client_sock:close(Sock2) + end), + + % duplicate client id + with_connection(fun([Sock, Sock1]) -> + emqx_zone:set_env(external, use_username_as_clientid, true), + emqx_client_sock:send(Sock, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + clean_start = true, + client_id = <<"myclient">>, + properties = + #{'Session-Expiry-Interval' => 10}}) + )), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0), _} = raw_recv_parse(Data, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock1, + raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + clean_start = false, + client_id = <<"myclient">>, + username = <<"admin">>, + password = <<"public">>, + properties = + #{'Session-Expiry-Interval' => 10}}) + )), + {ok, Data1} = gen_tcp:recv(Sock1, 0), + {ok, ?CONNACK_PACKET(?RC_SUCCESS, 0), _} = raw_recv_parse(Data1, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock, raw_send_serialize(?SUBSCRIBE_PACKET(1, [{<<"TopicA">>, #{rh => 1, + qos => ?QOS_2, + rap => 0, + nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5})), + + {ok, SubData} = gen_tcp:recv(Sock, 0), + {ok, ?SUBACK_PACKET(1, #{}, [2]), _} = raw_recv_parse(SubData, ?MQTT_PROTO_V5), + + emqx_client_sock:send(Sock1, raw_send_serialize(?SUBSCRIBE_PACKET(1, [{<<"TopicA">>, #{rh => 1, + qos => ?QOS_2, + rap => 0, + nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5})), + + {ok, SubData1} = gen_tcp:recv(Sock1, 0), + {ok, ?SUBACK_PACKET(1, #{}, [2]), _} = raw_recv_parse(SubData1, ?MQTT_PROTO_V5) + end, 2), + + ok. + +do_connect(Sock, ProtoVer) -> + emqx_client_sock:send(Sock, raw_send_serialize( + ?CONNECT_PACKET( + #mqtt_packet_connect{ + client_id = <<"mqtt_client">>, + proto_ver = ProtoVer + }))), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?CONNACK_PACKET(?CONNACK_ACCEPT), _} = raw_recv_parse(Data, ProtoVer). + +subscribe_v4(_) -> + with_connection(fun([Sock]) -> + do_connect(Sock, ?MQTT_PROTO_V4), + SubPacket = raw_send_serialize( + ?SUBSCRIBE_PACKET(15, + [{<<"topic">>, #{rh => 1, + qos => ?QOS_2, + rap => 0, + nl => 0, + rc => 0}}])), + emqx_client_sock:send(Sock, SubPacket), + {ok, Data} = gen_tcp:recv(Sock, 0), + {ok, ?SUBACK_PACKET(15, _), _} = raw_recv_parse(Data, ?MQTT_PROTO_V4) + end), + ok. + +subscribe_v5(_) -> + with_connection(fun([Sock]) -> + do_connect(Sock, ?MQTT_PROTO_V5), + SubPacket = raw_send_serialize(?SUBSCRIBE_PACKET(15, #{'Subscription-Identifier' => -1},[]), + #{version => ?MQTT_PROTO_V5}), + emqx_client_sock:send(Sock, SubPacket), + {ok, DisConnData} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_TOPIC_FILTER_INVALID), _} = + raw_recv_parse(DisConnData, ?MQTT_PROTO_V5) + end), + with_connection(fun([Sock]) -> + do_connect(Sock, ?MQTT_PROTO_V5), + SubPacket = raw_send_serialize( + ?SUBSCRIBE_PACKET(0, #{}, [{<<"TopicQos0">>, + #{rh => 1, qos => ?QOS_2, + rap => 0, nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5}), + emqx_client_sock:send(Sock, SubPacket), + {ok, DisConnData} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_MALFORMED_PACKET), _} = + raw_recv_parse(DisConnData, ?MQTT_PROTO_V5) + end), + with_connection(fun([Sock]) -> + do_connect(Sock, ?MQTT_PROTO_V5), + SubPacket = raw_send_serialize( + ?SUBSCRIBE_PACKET(1, #{'Subscription-Identifier' => 0}, + [{<<"TopicQos0">>, + #{rh => 1, qos => ?QOS_2, + rap => 0, nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5}), + emqx_client_sock:send(Sock, SubPacket), + {ok, DisConnData} = gen_tcp:recv(Sock, 0), + {ok, ?DISCONNECT_PACKET(?RC_SUBSCRIPTION_IDENTIFIERS_NOT_SUPPORTED), _} = + raw_recv_parse(DisConnData, ?MQTT_PROTO_V5) + end), + with_connection(fun([Sock]) -> + do_connect(Sock, ?MQTT_PROTO_V5), + SubPacket = raw_send_serialize( + ?SUBSCRIBE_PACKET(1, #{'Subscription-Identifier' => 1}, + [{<<"TopicQos0">>, + #{rh => 1, qos => ?QOS_2, + rap => 0, nl => 0, + rc => 0}}]), + #{version => ?MQTT_PROTO_V5}), + emqx_client_sock:send(Sock, SubPacket), + {ok, SubData} = gen_tcp:recv(Sock, 0), + {ok, ?SUBACK_PACKET(1, #{}, [2]), _} = + raw_recv_parse(SubData, ?MQTT_PROTO_V5) + end), + ok. + +publish_v4(_) -> + ok. + +publish_v5(_) -> + ok. + +raw_send_serialize(Packet) -> + emqx_frame:serialize(Packet). + +raw_send_serialize(Packet, Opts) -> + emqx_frame:serialize(Packet, Opts). + +raw_recv_parse(P, ProtoVersion) -> + emqx_frame:parse(P, {none, #{max_packet_size => ?MAX_PACKET_SIZE, + version => ProtoVersion}}). + + +acl_deny_action(_) -> + emqx_zone:set_env(external, acl_deny_action, disconnect), + process_flag(trap_exit, true), + [acl_deny_do_disconnect(publish, QoS, <<"acl_deny_action">>) || QoS <- lists:seq(0, 2)], + [acl_deny_do_disconnect(subscribe, QoS, <<"acl_deny_action">>) || QoS <- lists:seq(0, 2)], + emqx_zone:set_env(external, acl_deny_action, ignore), + ok. + +will_check(_) -> + process_flag(trap_exit, true), + will_topic_check(0), + will_acl_check(0). + +will_topic_check(QoS) -> + {ok, Client} = emqx_client:start_link([{username, <<"emqx">>}, + {will_flag, true}, + {will_topic, <<"">>}, + {will_payload, <<"I have died">>}, + {will_qos, QoS}]), + try emqx_client:connect(Client) of + _ -> + ok + catch + exit : _Reason -> + false = is_process_alive(Client) + end. + +will_acl_check(QoS) -> + {ok, Client} = emqx_client:start_link([{username, <<"emqx">>}, + {will_flag, true}, + {will_topic, <<"acl_deny_action">>}, + {will_payload, <<"I have died">>}, + {will_qos, QoS}]), + try emqx_client:connect(Client) of + _ -> + ok + catch + exit : _Reason -> + false = is_process_alive(Client) + end. + +acl_deny_do_disconnect(publish, QoS, Topic) -> + {ok, Client} = emqx_client:start_link([{username, <<"emqx">>}]), + {ok, _} = emqx_client:connect(Client), + emqx_client:publish(Client, Topic, <<"test">>, QoS), + receive + {'EXIT', Client, _Reason} -> + false = is_process_alive(Client) + end; +acl_deny_do_disconnect(subscribe, QoS, Topic) -> + {ok, Client} = emqx_client:start_link([{username, <<"emqx">>}]), + {ok, _} = emqx_client:connect(Client), + try emqx_client:subscribe(Client, Topic, QoS) of + _ -> + ok + catch + exit : _Reason -> + false = is_process_alive(Client) + end. + +start_apps(App, SchemaFile, ConfigFile) -> + read_schema_configs(App, SchemaFile, ConfigFile), + set_special_configs(App), + application:ensure_all_started(App). + +read_schema_configs(App, SchemaFile, ConfigFile) -> + Schema = cuttlefish_schema:files([SchemaFile]), + Conf = conf_parse:file(ConfigFile), + NewConfig = cuttlefish_generator:map(Schema, Conf), + Vals = proplists:get_value(App, NewConfig, []), + [application:set_env(App, Par, Value) || {Par, Value} <- Vals]. + +set_special_configs(emqx) -> + application:set_env(emqx, enable_acl_cache, false), + application:set_env(emqx, plugins_loaded_file, + deps_path(emqx, "test/emqx_SUITE_data/loaded_plugins")), + application:set_env(emqx, acl_deny_action, disconnect), + application:set_env(emqx, acl_file, + deps_path(emqx, "test/emqx_access_SUITE_data/acl_deny_action.conf")); +set_special_configs(_App) -> + ok. + +deps_path(App, RelativePath) -> + %% Note: not lib_dir because etc dir is not sym-link-ed to _build dir + %% but priv dir is + Path0 = code:priv_dir(App), + Path = case file:read_link(Path0) of + {ok, Resolved} -> Resolved; + {error, _} -> Path0 + end, + filename:join([Path, "..", RelativePath]). + +local_path(RelativePath) -> + deps_path(emqx_auth_username, RelativePath). diff --git a/test/emqx_protocol_tests.erl b/test/emqx_protocol_tests.erl new file mode 100644 index 000000000..56b65e36a --- /dev/null +++ b/test/emqx_protocol_tests.erl @@ -0,0 +1,30 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_protocol_tests). + +-include_lib("eunit/include/eunit.hrl"). + +set_property_test() -> + ?assertEqual(#{test => test_property}, emqx_protocol:set_property(test, test_property, undefined)), + TestMap = #{test => test_property}, + ?assertEqual(#{test => test_property, test1 => test_property2}, + emqx_protocol:set_property(test1, test_property2, TestMap)), + ok. + +init_username_test() -> + ?assertEqual(<<"Peercert">>, + emqx_protocol:init_username(<<"Peercert">>, [{peer_cert_as_username, crt}])), + ?assertEqual(undefined, + emqx_protocol:init_username(undefined, [{peer_cert_as_username, undefined}])). diff --git a/test/emqx_reason_codes_tests.erl b/test/emqx_reason_codes_tests.erl new file mode 100644 index 000000000..a8e352c83 --- /dev/null +++ b/test/emqx_reason_codes_tests.erl @@ -0,0 +1,134 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License") +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_reason_codes_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-include("emqx_mqtt.hrl"). + +-import(lists, [seq/2, zip/2, foreach/2]). + +-define(MQTTV4_CODE_NAMES, [connection_acceptd, + unacceptable_protocol_version, + client_identifier_not_valid, + server_unavaliable, + malformed_username_or_password, + unauthorized_client, + unknown_error]). + +-define(MQTTV5_CODE_NAMES, [success, granted_qos1, granted_qos2, disconnect_with_will_message, + no_matching_subscribers, no_subscription_existed, continue_authentication, + re_authenticate, unspecified_error, malformed_Packet, protocol_error, + implementation_specific_error, unsupported_protocol_version, + client_identifier_not_valid, bad_username_or_password, not_authorized, + server_unavailable, server_busy, banned,server_shutting_down, + bad_authentication_method, keepalive_timeout, session_taken_over, + topic_filter_invalid, topic_name_invalid, packet_identifier_inuse, + packet_identifier_not_found, receive_maximum_exceeded, topic_alias_invalid, + packet_too_large, message_rate_too_high, quota_exceeded, + administrative_action, payload_format_invalid, retain_not_supported, + qos_not_supported, use_another_server, server_moved, + shared_subscriptions_not_supported, connection_rate_exceeded, + maximum_connect_time, subscription_identifiers_not_supported, + wildcard_subscriptions_not_supported, unknown_error]). + +-define(MQTTV5_CODES, [16#00, 16#01, 16#02, 16#04, 16#10, 16#11, 16#18, 16#19, 16#80, 16#81, 16#82, + 16#83, 16#84, 16#85, 16#86, 16#87, 16#88, 16#89, 16#8A, 16#8B, 16#8C, 16#8D, + 16#8E, 16#8F, 16#90, 16#91, 16#92, 16#93, 16#94, 16#95, 16#96, 16#97, 16#98, + 16#99, 16#9A, 16#9B, 16#9C, 16#9D, 16#9E, 16#9F, 16#A0, 16#A1, 16#A2, code]). + +-define(MQTTV5_TXT, [<<"Success">>, <<"Granted QoS 1">>, <<"Granted QoS 2">>, + <<"Disconnect with Will Message">>, <<"No matching subscribers">>, + <<"No subscription existed">>, <<"Continue authentication">>, + <<"Re-authenticate">>, <<"Unspecified error">>, <<"Malformed Packet">>, + <<"Protocol Error">>, <<"Implementation specific error">>, + <<"Unsupported Protocol Version">>, <<"Client Identifier not valid">>, + <<"Bad User Name or Password">>, <<"Not authorized">>, + <<"Server unavailable">>, <<"Server busy">>, <<"Banned">>, + <<"Server shutting down">>, <<"Bad authentication method">>, + <<"Keep Alive timeout">>, <<"Session taken over">>, + <<"Topic Filter invalid">>, <<"Topic Name invalid">>, + <<"Packet Identifier in use">>, <<"Packet Identifier not found">>, + <<"Receive Maximum exceeded">>, <<"Topic Alias invalid">>, + <<"Packet too large">>, <<"Message rate too high">>, <<"Quota exceeded">>, + <<"Administrative action">>, <<"Payload format invalid">>, + <<"Retain not supported">>, <<"QoS not supported">>, + <<"Use another server">>, <<"Server moved">>, + <<"Shared Subscriptions not supported">>, <<"Connection rate exceeded">>, + <<"Maximum connect time">>, <<"Subscription Identifiers not supported">>, + <<"Wildcard Subscriptions not supported">>, <<"Unknown error">>]). + +-define(COMPAT_CODES_V5, [16#80, 16#81, 16#82, 16#83, 16#84, 16#85, 16#86, 16#87, + 16#88, 16#89, 16#8A, 16#8B, 16#8C, 16#97, 16#9C, 16#9D, + 16#9F]). + +-define(COMPAT_CODES_V4, [?CONNACK_PROTO_VER, ?CONNACK_PROTO_VER, ?CONNACK_PROTO_VER, + ?CONNACK_PROTO_VER, ?CONNACK_PROTO_VER, + ?CONNACK_INVALID_ID, + ?CONNACK_CREDENTIALS, + ?CONNACK_AUTH, + ?CONNACK_SERVER, + ?CONNACK_SERVER, + ?CONNACK_AUTH, + ?CONNACK_SERVER, + ?CONNACK_AUTH, + ?CONNACK_SERVER, ?CONNACK_SERVER, ?CONNACK_SERVER, ?CONNACK_SERVER]). + +mqttv4_name_test() -> + (((codes_test(?MQTT_PROTO_V4)) + (seq(0,6))) + (?MQTTV4_CODE_NAMES)) + (fun emqx_reason_codes:name/2). + +mqttv5_name_test() -> + (((codes_test(?MQTT_PROTO_V5)) + (?MQTTV5_CODES)) + (?MQTTV5_CODE_NAMES)) + (fun emqx_reason_codes:name/2). + +text_test() -> + (((codes_test(?MQTT_PROTO_V5)) + (?MQTTV5_CODES)) + (?MQTTV5_TXT)) + (fun emqx_reason_codes:text/1). + +compat_test() -> + (((codes_test(connack)) + (?COMPAT_CODES_V5)) + (?COMPAT_CODES_V4)) + (fun emqx_reason_codes:compat/2), + (((codes_test(suback)) + ([0,1,2, 16#80])) + ([0,1,2, 16#80])) + (fun emqx_reason_codes:compat/2), + (((codes_test(unsuback)) + ([0, 1, 2])) + ([undefined, undefined, undefined])) + (fun emqx_reason_codes:compat/2). + +codes_test(AsistVar) -> + fun(CODES) -> + fun(NAMES) -> + fun(Procedure) -> + foreach(fun({Code, Result}) -> + ?assertEqual(Result, case erlang:fun_info(Procedure, name) of + {name, text} -> Procedure(Code); + {name, name} -> Procedure(Code, AsistVar); + {name, compat} -> Procedure(AsistVar, Code) + end) + end, zip(CODES, NAMES)) + end + end + end. diff --git a/test/emqx_router_SUITE.erl b/test/emqx_router_SUITE.erl new file mode 100644 index 000000000..c115fd0cd --- /dev/null +++ b/test/emqx_router_SUITE.erl @@ -0,0 +1,92 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_router_SUITE). + +-include("emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(R, emqx_router). + +all() -> + [{group, route}]. + +groups() -> + [{route, [sequence], + [t_add_delete, + t_do_add_delete, + t_match_routes, + t_has_routes]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +init_per_testcase(_TestCase, Config) -> + clear_tables(), + Config. + +end_per_testcase(_TestCase, _Config) -> + clear_tables(). + +t_add_delete(_) -> + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/+/b">>, node()), + ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), + + ?R:delete_route(<<"a/b/c">>), + ?R:delete_route(<<"a/+/b">>, node()), + ?assertEqual([], ?R:topics()). + +t_do_add_delete(_) -> + ?R:do_add_route(<<"a/b/c">>, node()), + ?R:do_add_route(<<"a/b/c">>, node()), + ?R:do_add_route(<<"a/+/b">>, node()), + ?assertEqual([<<"a/+/b">>, <<"a/b/c">>], lists:sort(?R:topics())), + + ?R:do_delete_route(<<"a/b/c">>, node()), + ?R:do_delete_route(<<"a/+/b">>), + ?assertEqual([], ?R:topics()). + +t_match_routes(_) -> + ?R:add_route(<<"a/b/c">>, node()), + ?R:add_route(<<"a/+/c">>, node()), + ?R:add_route(<<"a/b/#">>, node()), + ?R:add_route(<<"#">>, node()), + ?assertEqual([#route{topic = <<"#">>, dest = node()}, + #route{topic = <<"a/+/c">>, dest = node()}, + #route{topic = <<"a/b/#">>, dest = node()}, + #route{topic = <<"a/b/c">>, dest = node()}], + lists:sort(?R:match_routes(<<"a/b/c">>))), + ?R:delete_route(<<"a/b/c">>, node()), + ?R:delete_route(<<"a/+/c">>, node()), + ?R:delete_route(<<"a/b/#">>, node()), + ?R:delete_route(<<"#">>, node()), + ?assertEqual([], lists:sort(?R:match_routes(<<"a/b/c">>))). + +t_has_routes(_) -> + ?R:add_route(<<"devices/+/messages">>, node()), + ?assert(?R:has_routes(<<"devices/+/messages">>)), + ?R:delete_route(<<"devices/+/messages">>). + +clear_tables() -> + lists:foreach(fun mnesia:clear_table/1, [emqx_route, emqx_trie, emqx_trie_node]). + diff --git a/test/emqx_sequence_SUITE.erl b/test/emqx_sequence_SUITE.erl new file mode 100644 index 000000000..ab408b8e0 --- /dev/null +++ b/test/emqx_sequence_SUITE.erl @@ -0,0 +1,38 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sequence_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-import(emqx_sequence, [nextval/2, reclaim/2]). + +all() -> + [sequence_generate]. + +sequence_generate(_) -> + ok = emqx_sequence:create(seqtab), + ?assertEqual(1, nextval(seqtab, key)), + ?assertEqual(2, nextval(seqtab, key)), + ?assertEqual(3, nextval(seqtab, key)), + ?assertEqual(2, reclaim(seqtab, key)), + ?assertEqual(1, reclaim(seqtab, key)), + ?assertEqual(0, reclaim(seqtab, key)), + ?assertEqual(false, ets:member(seqtab, key)), + ?assertEqual(1, nextval(seqtab, key)), + ?assert(emqx_sequence:delete(seqtab)). + diff --git a/test/emqx_session_SUITE.erl b/test/emqx_session_SUITE.erl new file mode 100644 index 000000000..b8e0aedd3 --- /dev/null +++ b/test/emqx_session_SUITE.erl @@ -0,0 +1,69 @@ + +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_session_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +-include_lib("common_test/include/ct.hrl"). + +all() -> [ignore_loop, t_session_all]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +ignore_loop(_Config) -> + application:set_env(emqx, mqtt_ignore_loop_deliver, true), + {ok, Client} = emqx_client:start_link(), + {ok, _} = emqx_client:connect(Client), + TestTopic = <<"Self">>, + {ok, _, [2]} = emqx_client:subscribe(Client, TestTopic, qos2), + ok = emqx_client:publish(Client, TestTopic, <<"testmsg">>, 0), + {ok, _} = emqx_client:publish(Client, TestTopic, <<"testmsg">>, 1), + {ok, _} = emqx_client:publish(Client, TestTopic, <<"testmsg">>, 2), + ?assertEqual(0, length(emqx_client_SUITE:receive_messages(3))), + ok = emqx_client:disconnect(Client), + application:set_env(emqx, mqtt_ignore_loop_deliver, false). + +t_session_all(_) -> + emqx_zone:set_env(internal, idle_timeout, 100), + ClientId = <<"ClientId">>, + {ok, ConnPid} = emqx_mock_client:start_link(ClientId), + {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), + Message1 = emqx_message:make(<<"ClientId">>, 2, <<"topic">>, <<"hello">>), + emqx_session:subscribe(SPid, [{<<"topic">>, #{qos => 2}}]), + emqx_session:subscribe(SPid, [{<<"topic">>, #{qos => 1}}]), + timer:sleep(200), + [{<<"topic">>, _}] = emqx:subscriptions(SPid), + emqx_session:publish(SPid, 1, Message1), + timer:sleep(200), + {publish, 1, _} = emqx_mock_client:get_last_message(ConnPid), + Attrs = emqx_session:attrs(SPid), + Info = emqx_session:info(SPid), + Stats = emqx_session:stats(SPid), + ClientId = proplists:get_value(client_id, Attrs), + ClientId = proplists:get_value(client_id, Info), + 1 = proplists:get_value(subscriptions_count, Stats), + emqx_session:unsubscribe(SPid, [<<"topic">>]), + timer:sleep(200), + [] = emqx:subscriptions(SPid), + emqx_mock_client:close_session(ConnPid). diff --git a/test/emqx_shared_sub_SUITE.erl b/test/emqx_shared_sub_SUITE.erl new file mode 100644 index 000000000..dd0e20ad1 --- /dev/null +++ b/test/emqx_shared_sub_SUITE.erl @@ -0,0 +1,294 @@ + +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_shared_sub_SUITE). + +-export([all/0, init_per_suite/1, end_per_suite/1]). +-export([t_random_basic/1, + t_random/1, + t_round_robin/1, + t_sticky/1, + t_hash/1, + t_not_so_sticky/1, + t_no_connection_nack/1 + ]). + +-include("emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(wait(For, Timeout), wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). + +all() -> [t_random_basic, + t_random, + t_round_robin, + t_sticky, + t_hash, + t_not_so_sticky, + t_no_connection_nack + ]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +t_random_basic(_) -> + ok = ensure_config(random), + ClientId = <<"ClientId">>, + {ok, ConnPid} = emqx_mock_client:start_link(ClientId), + {ok, SPid} = emqx_mock_client:open_session(ConnPid, ClientId, internal), + Message1 = emqx_message:make(<<"ClientId">>, 2, <<"foo">>, <<"hello">>), + emqx_session:subscribe(SPid, [{<<"foo">>, #{qos => 2, share => <<"group1">>}}]), + %% wait for the subscription to show up + ?wait(subscribed(<<"group1">>, <<"foo">>, SPid), 1000), + PacketId = 1, + emqx_session:publish(SPid, PacketId, Message1), + ?wait(case emqx_mock_client:get_last_message(ConnPid) of + {publish, 1, _} -> true; + Other -> Other + end, 1000), + emqx_session:pubrec(SPid, PacketId, reasoncode), + emqx_session:pubcomp(SPid, PacketId, reasoncode), + emqx_mock_client:close_session(ConnPid), + ok. + +%% Start two subscribers share subscribe to "$share/g1/foo/bar" +%% Set 'sticky' dispatch strategy, send 1st message to find +%% out which member it picked, then close its connection +%% send the second message, the message should be 'nack'ed +%% by the sticky session and delivered to the 2nd session. +t_no_connection_nack(_) -> + ok = ensure_config(sticky), + Publisher = <<"publisher">>, + Subscriber1 = <<"Subscriber1">>, + Subscriber2 = <<"Subscriber2">>, + QoS = 1, + Group = <<"g1">>, + Topic = <<"foo/bar">>, + {ok, PubConnPid} = emqx_mock_client:start_link(Publisher), + {ok, SubConnPid1} = emqx_mock_client:start_link(Subscriber1), + {ok, SubConnPid2} = emqx_mock_client:start_link(Subscriber2), + %% allow session to persist after connection shutdown + Attrs = #{expiry_interval => timer:seconds(30)}, + {ok, P_Pid} = emqx_mock_client:open_session(PubConnPid, Publisher, internal, Attrs), + {ok, SPid1} = emqx_mock_client:open_session(SubConnPid1, Subscriber1, internal, Attrs), + {ok, SPid2} = emqx_mock_client:open_session(SubConnPid2, Subscriber2, internal, Attrs), + emqx_session:subscribe(SPid1, [{Topic, #{qos => QoS, share => Group}}]), + emqx_session:subscribe(SPid2, [{Topic, #{qos => QoS, share => Group}}]), + %% wait for the subscriptions to show up + ?wait(subscribed(Group, Topic, SPid1), 1000), + ?wait(subscribed(Group, Topic, SPid2), 1000), + MkPayload = fun(PacketId) -> iolist_to_binary(["hello-", integer_to_list(PacketId)]) end, + SendF = fun(PacketId) -> emqx_session:publish(P_Pid, PacketId, emqx_message:make(Publisher, QoS, Topic, MkPayload(PacketId))) end, + SendF(1), + Ref = make_ref(), + CasePid = self(), + Received = + fun(PacketId, ConnPid) -> + Payload = MkPayload(PacketId), + case emqx_mock_client:get_last_message(ConnPid) of + {publish, _, #message{payload = Payload}} -> + CasePid ! {Ref, PacketId, ConnPid}, + true; + _Other -> + false + end + end, + ?wait(Received(1, SubConnPid1) orelse Received(1, SubConnPid2), 1000), + %% This is the connection which was picked by broker to dispatch (sticky) for 1st message + ConnPid = receive {Ref, 1, Pid} -> Pid after 1000 -> error(timeout) end, + %% Now kill the connection, expect all following messages to be delivered to the other subscriber. + emqx_mock_client:stop(ConnPid), + %% sleep then make synced calls to session processes to ensure that + %% the connection pid's 'EXIT' message is propagated to the session process + %% also to be sure sessions are still alive + timer:sleep(5), + _ = emqx_session:info(SPid1), + _ = emqx_session:info(SPid2), + %% Now we know what is the other still alive connection + [TheOtherConnPid] = [SubConnPid1, SubConnPid2] -- [ConnPid], + %% Send some more messages + PacketIdList = lists:seq(2, 10), + lists:foreach(fun(Id) -> + SendF(Id), + ?wait(Received(Id, TheOtherConnPid), 1000) + end, PacketIdList), + %% clean up + emqx_mock_client:close_session(PubConnPid), + emqx_sm:close_session(SPid1), + emqx_sm:close_session(SPid2), + emqx_mock_client:close_session(TheOtherConnPid), + ok. + +t_random(_) -> + test_two_messages(random). + +t_round_robin(_) -> + test_two_messages(round_robin). + +t_sticky(_) -> + test_two_messages(sticky). + +t_hash(_) -> + test_two_messages(hash, false). + +%% if the original subscriber dies, change to another one alive +t_not_so_sticky(_) -> + ok = ensure_config(sticky), + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + {ok, ConnPid1} = emqx_mock_client:start_link(ClientId1), + {ok, ConnPid2} = emqx_mock_client:start_link(ClientId2), + {ok, SPid1} = emqx_mock_client:open_session(ConnPid1, ClientId1, internal), + {ok, SPid2} = emqx_mock_client:open_session(ConnPid2, ClientId2, internal), + Message1 = emqx_message:make(ClientId1, 0, <<"foo/bar">>, <<"hello1">>), + Message2 = emqx_message:make(ClientId1, 0, <<"foo/bar">>, <<"hello2">>), + emqx_session:subscribe(SPid1, [{<<"foo/bar">>, #{qos => 0, share => <<"group1">>}}]), + %% wait for the subscription to show up + ?wait(subscribed(<<"group1">>, <<"foo/bar">>, SPid1), 1000), + emqx_session:publish(SPid1, 1, Message1), + ?wait(case emqx_mock_client:get_last_message(ConnPid1) of + {publish, _, #message{payload = <<"hello1">>}} -> true; + Other -> Other + end, 1000), + emqx_mock_client:close_session(ConnPid1), + ?wait(not subscribed(<<"group1">>, <<"foo/bar">>, SPid1), 1000), + emqx_session:subscribe(SPid2, [{<<"foo/#">>, #{qos => 0, share => <<"group1">>}}]), + ?wait(subscribed(<<"group1">>, <<"foo/#">>, SPid2), 1000), + emqx_session:publish(SPid2, 2, Message2), + ?wait(case emqx_mock_client:get_last_message(ConnPid2) of + {publish, _, #message{payload = <<"hello2">>}} -> true; + Other -> Other + end, 1000), + emqx_mock_client:close_session(ConnPid2), + ?wait(not subscribed(<<"group1">>, <<"foo/#">>, SPid2), 1000), + ok. + +test_two_messages(Strategy) -> + test_two_messages(Strategy, _WithAck = true). + +test_two_messages(Strategy, WithAck) -> + ok = ensure_config(Strategy, WithAck), + Topic = <<"foo/bar">>, + ClientId1 = <<"ClientId1">>, + ClientId2 = <<"ClientId2">>, + {ok, ConnPid1} = emqx_mock_client:start_link(ClientId1), + {ok, ConnPid2} = emqx_mock_client:start_link(ClientId2), + {ok, SPid1} = emqx_mock_client:open_session(ConnPid1, ClientId1, internal), + {ok, SPid2} = emqx_mock_client:open_session(ConnPid2, ClientId2, internal), + Message1 = emqx_message:make(ClientId1, 0, Topic, <<"hello1">>), + Message2 = emqx_message:make(ClientId1, 0, Topic, <<"hello2">>), + emqx_session:subscribe(SPid1, [{Topic, #{qos => 0, share => <<"group1">>}}]), + emqx_session:subscribe(SPid2, [{Topic, #{qos => 0, share => <<"group1">>}}]), + %% wait for the subscription to show up + ?wait(subscribed(<<"group1">>, Topic, SPid1) andalso + subscribed(<<"group1">>, Topic, SPid2), 1000), + emqx_broker:publish(Message1), + Me = self(), + WaitF = fun(ExpectedPayload) -> + case last_message(ExpectedPayload, [ConnPid1, ConnPid2]) of + {true, Pid} -> + Me ! {subscriber, Pid}, + true; + Other -> + Other + end + end, + ?wait(WaitF(<<"hello1">>), 2000), + UsedSubPid1 = receive {subscriber, P1} -> P1 end, + emqx_broker:publish(Message2), + ?wait(WaitF(<<"hello2">>), 2000), + UsedSubPid2 = receive {subscriber, P2} -> P2 end, + case Strategy of + sticky -> ?assert(UsedSubPid1 =:= UsedSubPid2); + round_robin -> ?assert(UsedSubPid1 =/= UsedSubPid2); + hash -> ?assert(UsedSubPid1 =:= UsedSubPid2); + _ -> ok + end, + emqx_mock_client:close_session(ConnPid1), + emqx_mock_client:close_session(ConnPid2), + ok. + +last_message(_ExpectedPayload, []) -> <<"not yet?">>; +last_message(ExpectedPayload, [Pid | Pids]) -> + case emqx_mock_client:get_last_message(Pid) of + {publish, _, #message{payload = ExpectedPayload}} -> {true, Pid}; + _Other -> last_message(ExpectedPayload, Pids) + end. + +%%------------------------------------------------------------------------------ +%% help functions +%%------------------------------------------------------------------------------ + +ensure_config(Strategy) -> + ensure_config(Strategy, _AckEnabled = true). + +ensure_config(Strategy, AckEnabled) -> + application:set_env(?APPLICATION, shared_subscription_strategy, Strategy), + application:set_env(?APPLICATION, shared_dispatch_ack_enabled, AckEnabled), + ok. + +subscribed(Group, Topic, Pid) -> + lists:member(Pid, emqx_shared_sub:subscribers(Group, Topic)). + +wait_for(Fn, Ln, F, Timeout) -> + {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_loop(F, catch_call(F)) end), + wait_for_down(Fn, Ln, Timeout, Pid, Mref, false). + +wait_for_down(Fn, Ln, Timeout, Pid, Mref, Kill) -> + receive + {'DOWN', Mref, process, Pid, normal} -> + ok; + {'DOWN', Mref, process, Pid, {unexpected, Result}} -> + erlang:error({unexpected, Fn, Ln, Result}); + {'DOWN', Mref, process, Pid, {crashed, {C, E, S}}} -> + erlang:raise(C, {Fn, Ln, E}, S) + after + Timeout -> + case Kill of + true -> + erlang:demonitor(Mref, [flush]), + erlang:exit(Pid, kill), + erlang:error({Fn, Ln, timeout}); + false -> + Pid ! stop, + wait_for_down(Fn, Ln, Timeout, Pid, Mref, true) + end + end. + +wait_loop(_F, ok) -> exit(normal); +wait_loop(F, LastRes) -> + receive + stop -> erlang:exit(LastRes) + after + 100 -> + Res = catch_call(F), + wait_loop(F, Res) + end. + +catch_call(F) -> + try + case F() of + true -> ok; + Other -> {unexpected, Other} + end + catch + C : E : S -> + {crashed, {C, E, S}} + end. + diff --git a/test/emqx_sm_SUITE.erl b/test/emqx_sm_SUITE.erl new file mode 100644 index 000000000..b3ce70c82 --- /dev/null +++ b/test/emqx_sm_SUITE.erl @@ -0,0 +1,92 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_sm_SUITE). + +-include("emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(ATTRS, #{clean_start => true, + client_id => <<"client">>, + zone => internal, + username => <<"emqx">>, + expiry_interval => 0, + max_inflight => 0, + topic_alias_maximum => 0, + will_msg => undefined}). + +all() -> [{group, sm}]. + +groups() -> + [{sm, [non_parallel_tests], + [t_open_close_session, + t_resume_session, + t_discard_session, + t_register_unregister_session, + t_get_set_session_attrs, + t_get_set_session_stats, + t_lookup_session_pids]}]. + +init_per_suite(Config) -> + emqx_ct_broker_helpers:run_setup_steps(), + Config. + +end_per_suite(_Config) -> + emqx_ct_broker_helpers:run_teardown_steps(). + +t_open_close_session(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(ok, emqx_sm:close_session(SPid)). + +t_resume_session(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual({ok, SPid}, emqx_sm:resume_session(<<"client">>, ?ATTRS#{conn_pid => ClientPid})). + +t_discard_session(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client1">>), + {ok, _SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(ok, emqx_sm:discard_session(<<"client1">>)). + +t_register_unregister_session(_) -> + Pid = self(), + {ok, _ClientPid} = emqx_mock_client:start_link(<<"client">>), + ?assertEqual(ok, emqx_sm:register_session(<<"client">>)), + ?assertEqual(ok, emqx_sm:register_session(<<"client">>, Pid)), + ?assertEqual(ok, emqx_sm:unregister_session(<<"client">>)), + ?assertEqual(ok, emqx_sm:unregister_session(<<"client">>), Pid). + +t_get_set_session_attrs(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(true, emqx_sm:set_session_attrs(<<"client">>, [?ATTRS#{conn_pid => ClientPid}])), + ?assertEqual(true, emqx_sm:set_session_attrs(<<"client">>, SPid, [?ATTRS#{conn_pid => ClientPid}])), + [SAttr] = emqx_sm:get_session_attrs(<<"client">>, SPid), + ?assertEqual(<<"client">>, maps:get(client_id, SAttr)). + +t_get_set_session_stats(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual(true, emqx_sm:set_session_stats(<<"client">>, [{inflight, 10}])), + ?assertEqual(true, emqx_sm:set_session_stats(<<"client">>, SPid, [{inflight, 10}])), + ?assertEqual([{inflight, 10}], emqx_sm:get_session_stats(<<"client">>, SPid)). + +t_lookup_session_pids(_) -> + {ok, ClientPid} = emqx_mock_client:start_link(<<"client">>), + {ok, SPid} = emqx_sm:open_session(?ATTRS#{conn_pid => ClientPid}), + ?assertEqual([SPid], emqx_sm:lookup_session_pids(<<"client">>)). diff --git a/test/emqx_stats_tests.erl b/test/emqx_stats_tests.erl new file mode 100644 index 000000000..e8b5e82af --- /dev/null +++ b/test/emqx_stats_tests.erl @@ -0,0 +1,101 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_stats_tests). + +-include_lib("eunit/include/eunit.hrl"). + +get_state_test() -> + with_proc(fun() -> + SetConnsCount = emqx_stats:statsfun('connections/count'), + SetConnsCount(1), + 1 = emqx_stats:getstat('connections/count'), + emqx_stats:setstat('connections/count', 2), + 2 = emqx_stats:getstat('connections/count'), + emqx_stats:setstat('connections/count', 'connections/max', 3), + timer:sleep(100), + 3 = emqx_stats:getstat('connections/count'), + 3 = emqx_stats:getstat('connections/max'), + emqx_stats:setstat('connections/count', 'connections/max', 2), + timer:sleep(100), + 2 = emqx_stats:getstat('connections/count'), + 3 = emqx_stats:getstat('connections/max'), + SetConns = emqx_stats:statsfun('connections/count', 'connections/max'), + SetConns(4), + timer:sleep(100), + 4 = emqx_stats:getstat('connections/count'), + 4 = emqx_stats:getstat('connections/max'), + Conns = emqx_stats:getstats(), + 4 = proplists:get_value('connections/count', Conns), + 4 = proplists:get_value('connections/max', Conns) + end). + +update_interval_test() -> + TickMs = 200, + with_proc(fun() -> + SleepMs = TickMs * 2 + TickMs div 2, %% sleep for 2.5 ticks + emqx_stats:cancel_update(cm_stats), + UpdFun = fun() -> emqx_stats:setstat('connections/count', 1) end, + ok = emqx_stats:update_interval(stats_test, UpdFun), + timer:sleep(SleepMs), + ?assertEqual(1, emqx_stats:getstat('connections/count')) + end, TickMs). + +helper_test_() -> + TickMs = 200, + TestF = + fun(CbModule, CbFun) -> + SleepMs = TickMs + TickMs div 2, %% sleep for 1.5 ticks + Ref = make_ref(), + Tester = self(), + UpdFun = + fun() -> + CbModule:CbFun(), + Tester ! Ref, + ok + end, + ok = emqx_stats:update_interval(stats_test, UpdFun), + timer:sleep(SleepMs), + receive Ref -> ok after 2000 -> error(timeout) end + end, + MkTestFun = + fun(CbModule, CbFun) -> + fun() -> + with_proc(fun() -> TestF(CbModule, CbFun) end, TickMs) + end + end, + [{"emqx_broker", MkTestFun(emqx_broker, stats_fun)}, + {"emqx_sm", MkTestFun(emqx_sm, stats_fun)}, + {"emqx_router_helper", MkTestFun(emqx_router_helper, stats_fun)}, + {"emqx_cm", MkTestFun(emqx_cm, stats_fun)} + ]. + +with_proc(F) -> + {ok, _Pid} = emqx_stats:start_link(), + with_stop(F). + +with_proc(F, TickMs) -> + {ok, _Pid} = emqx_stats:start_link(#{tick_ms => TickMs}), + with_stop(F). + +with_stop(F) -> + try + %% make a synced call to the gen_server so we know it has + %% started running, hence it is safe to continue with less risk of race condition + ?assertEqual(ignored, gen_server:call(emqx_stats, ignored)), + F() + after + ok = emqx_stats:stop() + end. + diff --git a/test/emqx_tables_SUITE.erl b/test/emqx_tables_SUITE.erl new file mode 100644 index 000000000..1002c0a0b --- /dev/null +++ b/test/emqx_tables_SUITE.erl @@ -0,0 +1,26 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_tables_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +all() -> [t_new]. + +t_new(_) -> + ok = emqx_tables:new(test_table, [{read_concurrency, true}]), + ets:insert(test_table, {key, 100}), + ok = emqx_tables:new(test_table, [{read_concurrency, true}]), + 100 = ets:lookup_element(test_table, key, 2). diff --git a/test/emqx_time_SUITE.erl b/test/emqx_time_SUITE.erl new file mode 100644 index 000000000..470b9dfe7 --- /dev/null +++ b/test/emqx_time_SUITE.erl @@ -0,0 +1,27 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_time_SUITE). + +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-compile(nowarn_export_all). + +all() -> [t_time_now_to]. + +t_time_now_to(_) -> + emqx_time:seed(), + emqx_time:now_secs(), + emqx_time:now_ms(). diff --git a/test/emqttd_topic_SUITE.erl b/test/emqx_topic_SUITE.erl similarity index 75% rename from test/emqttd_topic_SUITE.erl rename to test/emqx_topic_SUITE.erl index 984d0b299..8ce3faf79 100644 --- a/test/emqttd_topic_SUITE.erl +++ b/test/emqx_topic_SUITE.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,24 +11,37 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_topic_SUITE). +-module(emqx_topic_SUITE). -include_lib("eunit/include/eunit.hrl"). %% CT -compile(export_all). +-compile(nowarn_export_all). --import(emqttd_topic, [wildcard/1, match/2, validate/1, triples/1, join/1, - words/1, systop/1, feed_var/3, parse/1, parse/2]). +-import(emqx_topic, [wildcard/1, match/2, validate/1, triples/1, join/1, + words/1, systop/1, feed_var/3, parse/1]). -define(N, 10000). -all() -> [t_wildcard, t_match, t_match2, t_match3, t_validate, t_triples, t_join, - t_words, t_systop, t_feed_var, t_sys_match, 't_#_match', - t_sigle_level_validate, t_sigle_level_match, t_match_perf, - t_triples_perf, t_parse]. +all() -> + [t_wildcard, + t_match, t_match2, t_match3, + t_validate, + t_triples, + t_join, + t_levels, + t_words, + t_systop, + t_feed_var, + t_sys_match, + 't_#_match', + t_sigle_level_validate, + t_sigle_level_match, + t_match_perf, + t_triples_perf, + t_parse]. t_wildcard(_) -> true = wildcard(<<"a/b/#">>), @@ -45,10 +57,10 @@ t_match(_) -> true = match(<<"abc">>, <<"+">>), true = match(<<"a/b/c">>, <<"a/b/c">>), false = match(<<"a/b/c">>, <<"a/c/d">>), - false = match(<<"$shared/x/y">>, <<"+">>), - false = match(<<"$shared/x/y">>, <<"+/x/y">>), - false = match(<<"$shared/x/y">>, <<"#">>), - false = match(<<"$shared/x/y">>, <<"+/+/#">>), + false = match(<<"$share/x/y">>, <<"+">>), + false = match(<<"$share/x/y">>, <<"+/x/y">>), + false = match(<<"$share/x/y">>, <<"#">>), + false = match(<<"$share/x/y">>, <<"+/+/#">>), false = match(<<"house/1/sensor/0">>, <<"house/+">>), false = match(<<"house">>, <<"house/+">>). @@ -65,10 +77,10 @@ t_match2(_) -> true = match(<<"abc">>, <<"+">>), true = match(<<"a/b/c">>, <<"a/b/c">>), false = match(<<"a/b/c">>, <<"a/c/d">>), - false = match(<<"$shared/x/y">>, <<"+">>), - false = match(<<"$shared/x/y">>, <<"+/x/y">>), - false = match(<<"$shared/x/y">>, <<"#">>), - false = match(<<"$shared/x/y">>, <<"+/+/#">>), + false = match(<<"$share/x/y">>, <<"+">>), + false = match(<<"$share/x/y">>, <<"+/x/y">>), + false = match(<<"$share/x/y">>, <<"#">>), + false = match(<<"$share/x/y">>, <<"+/+/#">>), false = match(<<"house/1/sensor/0">>, <<"house/+">>). t_match3(_) -> @@ -120,20 +132,22 @@ t_validate(_) -> true = validate({filter, <<"x">>}), true = validate({name, <<"x//y">>}), true = validate({filter, <<"sport/tennis/#">>}), - false = validate({name, <<>>}), - false = validate({name, long_topic()}), - false = validate({name, <<"abc/#">>}), - false = validate({filter, <<"abc/#/1">>}), - false = validate({filter, <<"abc/#xzy/+">>}), - false = validate({filter, <<"abc/xzy/+9827">>}), - false = validate({filter, <<"sport/tennis#">>}), - false = validate({filter, <<"sport/tennis/#/ranking">>}). + catch validate({name, <<>>}), + catch validate({name, long_topic()}), + catch validate({name, <<"abc/#">>}), + catch validate({filter, <<"abc/#/1">>}), + catch validate({filter, <<"abc/#xzy/+">>}), + catch validate({filter, <<"abc/xzy/+9827">>}), + catch validate({filter, <<"sport/tennis#">>}), + catch validate({filter, <<"sport/tennis/#/ranking">>}), + ok. t_sigle_level_validate(_) -> true = validate({filter, <<"+">>}), true = validate({filter, <<"+/tennis/#">>}), true = validate({filter, <<"sport/+/player1">>}), - false = validate({filter, <<"sport+">>}). + catch validate({filter, <<"sport+">>}), + ok. t_triples(_) -> Triples = [{root,<<"a">>,<<"a">>}, @@ -148,6 +162,9 @@ t_triples_perf(_) -> end), io:format("Time for triples: ~p(micro)", [Time/?N]). +t_levels(_) -> + ?assertEqual(4, emqx_topic:levels(<<"a/b/c/d">>)). + t_words(_) -> ['', <<"a">>, '+', '#'] = words(<<"/a/+/#">>), ['', <<"abkc">>, <<"19383">>, '+', <<"akakdkkdkak">>, '#'] = words(<<"/abkc/19383/+/akakdkkdkak/#">>), @@ -184,10 +201,10 @@ long_topic() -> iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 10000)]). t_parse(_) -> - ?assertEqual({<<"a/b/+/#">>, []}, parse(<<"a/b/+/#">>)), - ?assertEqual({<<"topic">>, [{share, '$queue'}]}, parse(<<"$queue/topic">>)), - ?assertEqual({<<"topic">>, [{share, <<"group">>}]}, parse(<<"$share/group/topic">>)), - ?assertEqual({<<"topic">>, [local]}, parse(<<"$local/topic">>)), - ?assertEqual({<<"topic">>, [{share, '$queue'}, local]}, parse(<<"$local/$queue/topic">>)), - ?assertEqual({<<"/a/b/c">>, [{share, <<"group">>}, local]}, parse(<<"$local/$share/group//a/b/c">>)). - + ?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)), + ?assertEqual({<<"topic">>, #{ share => <<"$queue">> }}, parse(<<"$queue/topic">>)), + ?assertEqual({<<"topic">>, #{ share => <<"group">>}}, parse(<<"$share/group/topic">>)), + ?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)), + ?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)), + ?assertEqual({<<"$local/$share/group/a/b/c">>, #{}}, parse(<<"$local/$share/group/a/b/c">>)), + ?assertEqual({<<"$fastlane/topic">>, #{}}, parse(<<"$fastlane/topic">>)). diff --git a/test/emqx_trie_SUITE.erl b/test/emqx_trie_SUITE.erl new file mode 100644 index 000000000..500fe3574 --- /dev/null +++ b/test/emqx_trie_SUITE.erl @@ -0,0 +1,136 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_trie_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(TRIE, emqx_trie). +-define(TRIE_TABS, [emqx_trie, emqx_trie_node]). + +all() -> + [t_mnesia, t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. + +init_per_suite(Config) -> + application:load(emqx), + ok = ekka:start(), + Config. + +end_per_suite(_Config) -> + ekka:stop(), + ekka_mnesia:ensure_stopped(), + ekka_mnesia:delete_schema(). + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + clear_tables(). + +t_mnesia(_) -> + ok = ?TRIE:mnesia(copy). + +t_insert(_) -> + TN = #trie_node{node_id = <<"sensor">>, + edge_count = 3, + topic = <<"sensor">>, + flags = undefined}, + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>), + ?TRIE:insert(<<"sensor">>), + ?TRIE:insert(<<"sensor">>), + ?TRIE:lookup(<<"sensor">>) + end, + ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)). + +t_match(_) -> + Machted = [<<"sensor/+/#">>, <<"sensor/#">>], + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/+/#">>), + ?TRIE:insert(<<"sensor/#">>), + ?TRIE:match(<<"sensor/1">>) + end, + ?assertEqual({atomic, Machted}, mnesia:transaction(Fun)). + +t_match2(_) -> + Matched = {[<<"+/+/#">>, <<"+/#">>, <<"#">>], []}, + Fun = fun() -> + ?TRIE:insert(<<"#">>), + ?TRIE:insert(<<"+/#">>), + ?TRIE:insert(<<"+/+/#">>), + {?TRIE:match(<<"a/b/c">>), + ?TRIE:match(<<"$SYS/broker/zenmq">>)} + end, + ?assertEqual({atomic, Matched}, mnesia:transaction(Fun)). + +t_match3(_) -> + Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], + mnesia:transaction(fun() -> [emqx_trie:insert(Topic) || Topic <- Topics] end), + Matched = mnesia:async_dirty(fun emqx_trie:match/1, [<<"a/b/c">>]), + ?assertEqual(4, length(Matched)), + SysMatched = mnesia:async_dirty(fun emqx_trie:match/1, [<<"$SYS/a/b/c">>]), + ?assertEqual([<<"$SYS/#">>], SysMatched). + +t_delete(_) -> + TN = #trie_node{node_id = <<"sensor/1">>, + edge_count = 2, + topic = undefined, + flags = undefined}, + Fun = fun() -> + ?TRIE:insert(<<"sensor/1/#">>), + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:delete(<<"sensor/1/metric/2">>), + ?TRIE:delete(<<"sensor/1/metric">>), + ?TRIE:delete(<<"sensor/1/metric">>), + ?TRIE:lookup(<<"sensor/1">>) + end, + ?assertEqual({atomic, [TN]}, mnesia:transaction(Fun)). + +t_delete2(_) -> + Fun = fun() -> + ?TRIE:insert(<<"sensor">>), + ?TRIE:insert(<<"sensor/1/metric/2">>), + ?TRIE:insert(<<"sensor/1/metric/3">>), + ?TRIE:delete(<<"sensor">>), + ?TRIE:delete(<<"sensor/1/metric/2">>), + ?TRIE:delete(<<"sensor/1/metric/3">>), + {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/1">>)} + end, + ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)). + +t_delete3(_) -> + Fun = fun() -> + ?TRIE:insert(<<"sensor/+">>), + ?TRIE:insert(<<"sensor/+/metric/2">>), + ?TRIE:insert(<<"sensor/+/metric/3">>), + ?TRIE:delete(<<"sensor/+/metric/2">>), + ?TRIE:delete(<<"sensor/+/metric/3">>), + ?TRIE:delete(<<"sensor">>), + ?TRIE:delete(<<"sensor/+">>), + ?TRIE:delete(<<"sensor/+/unknown">>), + {?TRIE:lookup(<<"sensor">>), ?TRIE:lookup(<<"sensor/+">>)} + end, + ?assertEqual({atomic, {[], []}}, mnesia:transaction(Fun)). + +clear_tables() -> + lists:foreach(fun mnesia:clear_table/1, ?TRIE_TABS). + diff --git a/test/emqttd_vm_SUITE.erl b/test/emqx_vm_SUITE.erl similarity index 82% rename from test/emqttd_vm_SUITE.erl rename to test/emqx_vm_SUITE.erl index 49252c2ae..91744764e 100644 --- a/test/emqttd_vm_SUITE.erl +++ b/test/emqx_vm_SUITE.erl @@ -1,5 +1,4 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2013-2018 EMQ Enterprise, Inc. (http://emqtt.io) +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -12,11 +11,11 @@ %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. -%%-------------------------------------------------------------------- --module(emqttd_vm_SUITE). +-module(emqx_vm_SUITE). -compile(export_all). +-compile(nowarn_export_all). -include_lib("common_test/include/ct.hrl"). @@ -96,79 +95,79 @@ all() -> - [load, systeminfo, mem_info, process_list, process_info, process_gc, + [load, systeminfo, mem_info, process_list, process_info, process_gc, get_ets_list, get_ets_info, get_ets_object, get_port_types, get_port_info, scheduler_usage, get_memory, microsecs, schedulers, get_process_group_leader_info, get_process_limit]. load(_Config) -> - Loads = emqttd_vm:loads(), + Loads = emqx_vm:loads(), [{load1, _}, {load5, _}, {load15, _}] = Loads. systeminfo(_Config) -> - Keys = [Key || {Key, _} <- emqttd_vm:get_system_info()], + Keys = [Key || {Key, _} <- emqx_vm:get_system_info()], ?SYSTEM_INFO = Keys. mem_info(_Config) -> application:ensure_all_started(os_mon), - MemInfo = emqttd_vm:mem_info(), + MemInfo = emqx_vm:mem_info(), [{total_memory, _}, {used_memory, _}]= MemInfo, application:stop(os_mon). process_list(_Config) -> Pid = self(), - ProcessInfo = emqttd_vm:get_process_list(), + ProcessInfo = emqx_vm:get_process_list(), true = lists:member({pid, Pid}, lists:concat(ProcessInfo)). process_info(_Config) -> - ProcessInfos = emqttd_vm:get_process_info(), + ProcessInfos = emqx_vm:get_process_info(), ProcessInfo = lists:last(ProcessInfos), Keys = [K || {K, _V}<- ProcessInfo], ?PROCESS_INFO = Keys. process_gc(_Config) -> - ProcessGcs = emqttd_vm:get_process_gc(), + ProcessGcs = emqx_vm:get_process_gc(), ProcessGc = lists:last(ProcessGcs), Keys = [K || {K, _V}<- ProcessGc], ?PROCESS_GC = Keys. get_ets_list(_Config) -> ets:new(test, [named_table]), - Ets = emqttd_vm:get_ets_list(), + Ets = emqx_vm:get_ets_list(), true = lists:member(test, Ets). -get_ets_info(_Config) -> +get_ets_info(_Config) -> ets:new(test, [named_table]), - [] = emqttd_vm:get_ets_info(test1), - EtsInfo = emqttd_vm:get_ets_info(test), + [] = emqx_vm:get_ets_info(test1), + EtsInfo = emqx_vm:get_ets_info(test), test = proplists:get_value(name, EtsInfo). get_ets_object(_Config) -> ets:new(test, [named_table]), ets:insert(test, {k, v}), - [{k, v}] = emqttd_vm:get_ets_object(test). + [{k, v}] = emqx_vm:get_ets_object(test). get_port_types(_Config) -> - emqttd_vm:get_port_types(). + emqx_vm:get_port_types(). get_port_info(_Config) -> - emqttd_vm:get_port_info(). + emqx_vm:get_port_info(). scheduler_usage(_Config) -> - emqttd_vm:scheduler_usage(5000). + emqx_vm:scheduler_usage(5000). get_memory(_Config) -> - emqttd_vm:get_memory(). + emqx_vm:get_memory(). microsecs(_Config) -> - emqttd_vm:microsecs(). + emqx_vm:microsecs(). schedulers(_Config) -> - emqttd_vm:schedulers(). + emqx_vm:schedulers(). get_process_group_leader_info(_Config) -> - emqttd_vm:get_process_group_leader_info(self()). + emqx_vm:get_process_group_leader_info(self()). get_process_limit(_Config) -> - emqttd_vm:get_process_limit(). + emqx_vm:get_process_limit(). diff --git a/test/emqx_zone_SUITE.erl b/test/emqx_zone_SUITE.erl new file mode 100644 index 000000000..83c2ceaab --- /dev/null +++ b/test/emqx_zone_SUITE.erl @@ -0,0 +1,38 @@ +%% Copyright (c) 2018 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +-module(emqx_zone_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_mqtt.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +all() -> [t_set_get_env]. + +t_set_get_env(_) -> + application:set_env(emqx, zones, [{china, [{language, chinese}]}]), + {ok, _} = emqx_zone:start_link(), + ct:print("~p~n", [ets:tab2list(emqx_zone)]), + chinese = emqx_zone:get_env(china, language), + cn470 = emqx_zone:get_env(china, ism_band, cn470), + undefined = emqx_zone:get_env(undefined, delay), + 500 = emqx_zone:get_env(undefined, delay, 500), + application:set_env(emqx, zones, [{zone1, [{key, val}]}]), + ?assertEqual(undefined, emqx_zone:get_env(zone1, key)), + emqx_zone:force_reload(), + ?assertEqual(val, emqx_zone:get_env(zone1, key)), + emqx_zone:stop(). + diff --git a/test/rfc6455_client.erl b/test/rfc6455_client.erl new file mode 100644 index 000000000..f5d8f1ef4 --- /dev/null +++ b/test/rfc6455_client.erl @@ -0,0 +1,252 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License at +%% http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +%% License for the specific language governing rights and limitations +%% under the License. +%% +%% The Original Code is RabbitMQ Management Console. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2012-2016 Pivotal Software, Inc. All rights reserved. +%% + +-module(rfc6455_client). + +-export([new/2, open/1, recv/1, send/2, send_binary/2, close/1, close/2]). + +-record(state, {host, port, addr, path, ppid, socket, data, phase}). + +%% -------------------------------------------------------------------------- + +new(WsUrl, PPid) -> + crypto:start(), + "ws://" ++ Rest = WsUrl, + [Addr, Path] = split("/", Rest, 1), + [Host, MaybePort] = split(":", Addr, 1, empty), + Port = case MaybePort of + empty -> 80; + V -> {I, ""} = string:to_integer(V), I + end, + State = #state{host = Host, + port = Port, + addr = Addr, + path = "/" ++ Path, + ppid = PPid}, + spawn(fun () -> + start_conn(State) + end). + +open(WS) -> + receive + {rfc6455, open, WS, Opts} -> + {ok, Opts}; + {rfc6455, close, WS, R} -> + {close, R} + end. + +recv(WS) -> + receive + {rfc6455, recv, WS, Payload} -> + {ok, Payload}; + {rfc6455, recv_binary, WS, Payload} -> + {binary, Payload}; + {rfc6455, close, WS, R} -> + {close, R} + end. + +send(WS, IoData) -> + WS ! {send, IoData}, + ok. + +send_binary(WS, IoData) -> + WS ! {send_binary, IoData}, + ok. + +close(WS) -> + close(WS, {1000, ""}). + +close(WS, WsReason) -> + WS ! {close, WsReason}, + receive + {rfc6455, close, WS, R} -> + {close, R} + end. + + +%% -------------------------------------------------------------------------- + +start_conn(State) -> + {ok, Socket} = gen_tcp:connect(State#state.host, State#state.port, + [binary, + {packet, 0}]), + Key = base64:encode_to_string(crypto:strong_rand_bytes(16)), + gen_tcp:send(Socket, + "GET " ++ State#state.path ++ " HTTP/1.1\r\n" ++ + "Host: " ++ State#state.addr ++ "\r\n" ++ + "Upgrade: websocket\r\n" ++ + "Connection: Upgrade\r\n" ++ + "Sec-WebSocket-Key: " ++ Key ++ "\r\n" ++ + "Origin: null\r\n" ++ + "Sec-WebSocket-Protocol: mqtt\r\n" ++ + "Sec-WebSocket-Version: 13\r\n\r\n"), + + loop(State#state{socket = Socket, + data = <<>>, + phase = opening}). + +do_recv(State = #state{phase = opening, ppid = PPid, data = Data}) -> + case split("\r\n\r\n", binary_to_list(Data), 1, empty) of + [_Http, empty] -> State; + [Http, Data1] -> + %% TODO: don't ignore http response data, verify key + PPid ! {rfc6455, open, self(), [{http_response, Http}]}, + State#state{phase = open, + data = Data1} + end; +do_recv(State = #state{phase = Phase, data = Data, socket = Socket, ppid = PPid}) + when Phase =:= open orelse Phase =:= closing -> + R = case Data of + <> + when L < 126 -> + {F, O, Payload, Rest}; + + <> -> + {F, O, Payload, Rest}; + + <> -> + {F, O, Payload, Rest}; + + <<_:1, _:3, _:4, 1:1, _/binary>> -> + %% According o rfc6455 5.1 the server must not mask any frames. + die(Socket, PPid, {1006, "Protocol error"}, normal); + _ -> + moredata + end, + case R of + moredata -> + State; + _ -> do_recv2(State, R) + end. + +do_recv2(State = #state{phase = Phase, socket = Socket, ppid = PPid}, R) -> + case R of + {1, 1, Payload, Rest} -> + PPid ! {rfc6455, recv, self(), Payload}, + State#state{data = Rest}; + {1, 2, Payload, Rest} -> + PPid ! {rfc6455, recv_binary, self(), Payload}, + State#state{data = Rest}; + {1, 8, Payload, _Rest} -> + WsReason = case Payload of + <> -> {WC, WR}; + <<>> -> {1005, "No status received"} + end, + case Phase of + open -> %% echo + do_close(State, WsReason), + gen_tcp:close(Socket); + closing -> + ok + end, + die(Socket, PPid, WsReason, normal); + {_, _, _, _Rest2} -> + io:format("Unknown frame type~n"), + die(Socket, PPid, {1006, "Unknown frame type"}, normal) + end. + +encode_frame(F, O, Payload) -> + Mask = crypto:strong_rand_bytes(4), + MaskedPayload = apply_mask(Mask, iolist_to_binary(Payload)), + + L = byte_size(MaskedPayload), + IoData = case L of + _ when L < 126 -> + [<>, Mask, MaskedPayload]; + _ when L < 65536 -> + [<>, Mask, MaskedPayload]; + _ -> + [<>, Mask, MaskedPayload] + end, + iolist_to_binary(IoData). + +do_send(State = #state{socket = Socket}, Payload) -> + gen_tcp:send(Socket, encode_frame(1, 1, Payload)), + State. + +do_send_binary(State = #state{socket = Socket}, Payload) -> + gen_tcp:send(Socket, encode_frame(1, 2, Payload)), + State. + +do_close(State = #state{socket = Socket}, {Code, Reason}) -> + Payload = iolist_to_binary([<>, Reason]), + gen_tcp:send(Socket, encode_frame(1, 8, Payload)), + State#state{phase = closing}. + + +loop(State = #state{socket = Socket, ppid = PPid, data = Data, + phase = Phase}) -> + receive + {tcp, Socket, Bin} -> + State1 = State#state{data = iolist_to_binary([Data, Bin])}, + loop(do_recv(State1)); + {send, Payload} when Phase == open -> + loop(do_send(State, Payload)); + {send_binary, Payload} when Phase == open -> + loop(do_send_binary(State, Payload)); + {tcp_closed, Socket} -> + die(Socket, PPid, {1006, "Connection closed abnormally"}, normal); + {close, WsReason} when Phase == open -> + loop(do_close(State, WsReason)) + end. + + +die(Socket, PPid, WsReason, Reason) -> + gen_tcp:shutdown(Socket, read_write), + PPid ! {rfc6455, close, self(), WsReason}, + exit(Reason). + + +%% -------------------------------------------------------------------------- + +split(SubStr, Str, Limit) -> + split(SubStr, Str, Limit, ""). + +split(SubStr, Str, Limit, Default) -> + Acc = split(SubStr, Str, Limit, [], Default), + lists:reverse(Acc). +split(_SubStr, Str, 0, Acc, _Default) -> [Str | Acc]; +split(SubStr, Str, Limit, Acc, Default) -> + {L, R} = case string:str(Str, SubStr) of + 0 -> {Str, Default}; + I -> {string:substr(Str, 1, I-1), + string:substr(Str, I+length(SubStr))} + end, + split(SubStr, R, Limit-1, [L | Acc], Default). + + +apply_mask(Mask, Data) when is_number(Mask) -> + apply_mask(<>, Data); + +apply_mask(<<0:32>>, Data) -> + Data; +apply_mask(Mask, Data) -> + iolist_to_binary(lists:reverse(apply_mask2(Mask, Data, []))). + +apply_mask2(M = <>, <>, Acc) -> + T = Data bxor Mask, + apply_mask2(M, Rest, [<> | Acc]); +apply_mask2(<>, <>, Acc) -> + T = Data bxor Mask, + [<> | Acc]; +apply_mask2(<>, <>, Acc) -> + T = Data bxor Mask, + [<> | Acc]; +apply_mask2(<>, <>, Acc) -> + T = Data bxor Mask, + [<> | Acc]; +apply_mask2(_, <<>>, Acc) -> + Acc. diff --git a/test/ws_client.erl b/test/ws_client.erl new file mode 100644 index 000000000..39f01467a --- /dev/null +++ b/test/ws_client.erl @@ -0,0 +1,75 @@ +-module(ws_client). + +-export([ + start_link/0, + start_link/1, + send_binary/2, + send_ping/2, + recv/2, + recv/1, + stop/1 + ]). + +-export([ + init/2, + websocket_handle/3, + websocket_info/3, + websocket_terminate/3 + ]). + +-record(state, { + buffer = [] :: list(), + waiting = undefined :: undefined | pid() + }). + +start_link() -> + start_link("ws://localhost:8083/mqtt"). + +start_link(Url) -> + websocket_client:start_link(Url, ?MODULE, [], [{extra_headers, [{"Sec-Websocket-Protocol", "mqtt"}]}]). + +stop(Pid) -> + Pid ! stop. + +send_binary(Pid, Msg) -> + websocket_client:cast(Pid, {binary, Msg}). + +send_ping(Pid, Msg) -> + websocket_client:cast(Pid, {ping, Msg}). + +recv(Pid) -> + recv(Pid, 5000). + +recv(Pid, Timeout) -> + Pid ! {recv, self()}, + receive + M -> M + after + Timeout -> error + end. + +init(_, _WSReq) -> + {ok, #state{}}. + +websocket_handle(Frame, _, State = #state{waiting = undefined, buffer = Buffer}) -> + logger:info("Client received frame~p", [Frame]), + {ok, State#state{buffer = [Frame|Buffer]}}; +websocket_handle(Frame, _, State = #state{waiting = From}) -> + logger:info("Client received frame~p", [Frame]), + From ! Frame, + {ok, State#state{waiting = undefined}}. + +websocket_info({send_text, Text}, WSReq, State) -> + websocket_client:send({text, Text}, WSReq), + {ok, State}; +websocket_info({recv, From}, _, State = #state{buffer = []}) -> + {ok, State#state{waiting = From}}; +websocket_info({recv, From}, _, State = #state{buffer = [Top|Rest]}) -> + From ! Top, + {ok, State#state{buffer = Rest}}; +websocket_info(stop, _, State) -> + {close, <<>>, State}. + +websocket_terminate(Close, _, State) -> + io:format("Websocket closed with frame ~p and state ~p", [Close, State]), + ok. diff --git a/vars b/vars new file mode 100644 index 000000000..fedd69a45 --- /dev/null +++ b/vars @@ -0,0 +1,9 @@ +%% vars here are for test only, not intended for release + +{platform_bin_dir, "bin"}. +{platform_data_dir, "data"}. +{platform_etc_dir, "etc"}. +{platform_lib_dir, "lib"}. +{platform_log_dir, "log"}. +{platform_plugins_dir, "plugins"}. +