diff --git a/.ci/docker-compose-file/.env b/.ci/docker-compose-file/.env index a71e174d9..e3beb1bc7 100644 --- a/.ci/docker-compose-file/.env +++ b/.ci/docker-compose-file/.env @@ -4,5 +4,6 @@ MONGO_TAG=5 PGSQL_TAG=13 LDAP_TAG=2.4.50 INFLUXDB_TAG=2.5.0 +TDENGINE_TAG=3.0.2.4 TARGET=emqx/emqx diff --git a/.ci/docker-compose-file/Makefile.local b/.ci/docker-compose-file/Makefile.local deleted file mode 100644 index 9c12255e4..000000000 --- a/.ci/docker-compose-file/Makefile.local +++ /dev/null @@ -1,64 +0,0 @@ -.PHONY: help up down ct ct-all bash run - -define usage -make -f .ci/docker-compose-file/Makefile.local up -make -f .ci/docker-compose-file/Makefile.local ct CONTAINER=erlang SUITE=apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl -make -f .ci/docker-compose-file/Makefile.local down -endef -export usage - -help: - @echo "$$usage" - -up: - env \ - MYSQL_TAG=8 \ - REDIS_TAG=7.0 \ - MONGO_TAG=5 \ - PGSQL_TAG=13 \ - docker-compose \ - -f .ci/docker-compose-file/docker-compose.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ - up -d --build --remove-orphans - -down: - docker-compose \ - -f .ci/docker-compose-file/docker-compose.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mongo-single-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-single-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-sentinel-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-sentinel-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-cluster-tcp.yaml \ - -f .ci/docker-compose-file/docker-compose-redis-cluster-tls.yaml \ - -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ - down --remove-orphans - -ct: - docker exec -i "$(CONTAINER)" bash -c "rebar3 ct --name 'test@127.0.0.1' --readable true -v --suite $(SUITE)" - -ct-all: - docker exec -i "$(CONTAINER)" bash -c "make ct" - -bash: - docker exec -it "$(CONTAINER)" bash - -run: - docker exec -it "$(CONTAINER)" bash -c "make run"; diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index 9662b174d..976b0bc1c 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -19,7 +19,7 @@ services: command: /bin/generate-certs.sh kdc: hostname: kdc.emqx.net - image: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04 container_name: kdc.emqx.net networks: emqx_bridge: @@ -39,9 +39,12 @@ services: container_name: kafka-1.emqx.net hostname: kafka-1.emqx.net depends_on: - - "kdc" - - "zookeeper" - - "ssl_cert_gen" + kdc: + condition: service_started + zookeeper: + condition: service_started + ssl_cert_gen: + condition: service_completed_successfully environment: KAFKA_BROKER_ID: 1 KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 @@ -52,7 +55,7 @@ services: KAFKA_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512,GSSAPI KAFKA_SASL_KERBEROS_SERVICE_NAME: kafka KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN - KAFKA_JMX_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" + KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/jaas.conf" KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" KAFKA_CREATE_TOPICS_NG: test-topic-one-partition:1:1,test-topic-two-partitions:2:1,test-topic-three-partitions:3:1, KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer diff --git a/.ci/docker-compose-file/docker-compose-tdengine-restful.yaml b/.ci/docker-compose-file/docker-compose-tdengine-restful.yaml new file mode 100644 index 000000000..6cd7f2669 --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-tdengine-restful.yaml @@ -0,0 +1,11 @@ +version: '3.9' + +services: + tdengine_server: + container_name: tdengine + image: tdengine/tdengine:${TDENGINE_TAG} + restart: always + ports: + - "6041:6041" + networks: + - emqx_bridge diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index ce4f28ba7..3f526978e 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -17,6 +17,7 @@ services: - 13307:3307 - 15432:5432 - 15433:5433 + - 16041:6041 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index b55b3196f..ff330872d 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang: container_name: erlang - image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04} + image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04} env_file: - conf.env environment: diff --git a/.ci/docker-compose-file/pgsql/Dockerfile b/.ci/docker-compose-file/pgsql/Dockerfile index f26e18d0e..8598d3f33 100644 --- a/.ci/docker-compose-file/pgsql/Dockerfile +++ b/.ci/docker-compose-file/pgsql/Dockerfile @@ -1,7 +1,7 @@ ARG BUILD_FROM=postgres:13 FROM ${BUILD_FROM} ARG POSTGRES_USER=postgres -COPY --chown=$POSTGRES_USER ./pgsql/pg_hba.conf /var/lib/postgresql/pg_hba.conf +COPY --chown=$POSTGRES_USER ./pgsql/pg_hba_tls.conf /var/lib/postgresql/pg_hba.conf COPY --chown=$POSTGRES_USER certs/server.key /var/lib/postgresql/server.key COPY --chown=$POSTGRES_USER certs/server.crt /var/lib/postgresql/server.crt COPY --chown=$POSTGRES_USER certs/ca.crt /var/lib/postgresql/root.crt diff --git a/.ci/docker-compose-file/pgsql/pg_hba_tls.conf b/.ci/docker-compose-file/pgsql/pg_hba_tls.conf new file mode 100644 index 000000000..356afd9a6 --- /dev/null +++ b/.ci/docker-compose-file/pgsql/pg_hba_tls.conf @@ -0,0 +1,8 @@ +# TYPE DATABASE USER CIDR-ADDRESS METHOD +local all all trust +# TODO: also test with `cert`? will require client certs +hostssl all all 0.0.0.0/0 password +hostssl all all ::/0 password + +hostssl all www-data 0.0.0.0/0 cert clientcert=1 +hostssl all postgres 0.0.0.0/0 cert clientcert=1 diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index f4b11116b..e26134ec8 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -41,5 +41,11 @@ "listen": "0.0.0.0:5433", "upstream": "pgsql-tls:5432", "enabled": true + }, + { + "name": "tdengine_restful", + "listen": "0.0.0.0:6041", + "upstream": "tdengine:6041", + "enabled": true } ] diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 95915dd7d..49d9b3dbf 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -3,7 +3,7 @@ inputs: profile: # emqx, emqx-enterprise required: true type: string - otp: # 25.1.2-2, 24.3.4.2-1 + otp: # 25.1.2-2, 24.3.4.2-2 required: true type: string os: diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index b21683dc7..c612d2d5f 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -25,7 +25,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04" outputs: PROFILE: ${{ steps.get_profile.outputs.PROFILE }} @@ -125,9 +125,9 @@ jobs: # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix builder: - - 5.0-27 # update to latest + - 5.0-28 # update to latest otp: - - 24.3.4.2-1 # switch to 25 once ready to release 5.1 + - 24.3.4.2-2 # switch to 25 once ready to release 5.1 elixir: - 'no_elixir' - '1.13.4' # update to latest diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 6abb36d86..b23e2c604 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -23,7 +23,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -150,7 +150,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.3.4.2-1 + - 24.3.4.2-2 os: - macos-11 - macos-12-arm64 @@ -201,7 +201,7 @@ jobs: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} otp: - - 24.3.4.2-1 + - 24.3.4.2-2 arch: - amd64 - arm64 @@ -218,7 +218,7 @@ jobs: - aws-arm64 - ubuntu-20.04 builder: - - 5.0-27 + - 5.0-28 elixir: - 1.13.4 exclude: @@ -232,7 +232,7 @@ jobs: arch: amd64 os: ubuntu22.04 build_machine: ubuntu-22.04 - builder: 5.0-27 + builder: 5.0-28 elixir: 1.13.4 release_with: elixir - profile: emqx @@ -240,7 +240,7 @@ jobs: arch: amd64 os: amzn2 build_machine: ubuntu-22.04 - builder: 5.0-27 + builder: 5.0-28 elixir: 1.13.4 release_with: elixir diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index c7b5b0c83..692e4a987 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -29,13 +29,13 @@ jobs: fail-fast: false matrix: profile: - - ["emqx", "24.3.4.2-1", "el7"] - - ["emqx", "24.3.4.2-1", "ubuntu20.04"] + - ["emqx", "24.3.4.2-2", "el7"] + - ["emqx", "24.3.4.2-2", "ubuntu20.04"] - ["emqx", "25.1.2-2", "ubuntu22.04"] - - ["emqx-enterprise", "24.3.4.2-1", "ubuntu20.04"] + - ["emqx-enterprise", "24.3.4.2-2", "ubuntu20.04"] - ["emqx-enterprise", "25.1.2-2", "ubuntu22.04"] builder: - - 5.0-27 + - 5.0-28 elixir: - 1.13.4 @@ -128,7 +128,7 @@ jobs: - emqx - emqx-enterprise otp: - - 24.3.4.2-1 + - 24.3.4.2-2 os: - macos-11 - macos-12-arm64 @@ -154,6 +154,50 @@ jobs: name: ${{ matrix.os }} path: _packages/**/* + docker: + runs-on: ubuntu-22.04 + + strategy: + fail-fast: false + matrix: + profile: + - emqx + - emqx-enterprise + + steps: + - uses: actions/checkout@v3 + - name: prepare + run: | + EMQX_NAME=${{ matrix.profile }} + PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)} + EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test + echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV + echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV + echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV + - uses: docker/setup-buildx-action@v2 + - name: build and export to Docker + uses: docker/build-push-action@v4 + with: + context: . + file: ./deploy/docker/Dockerfile + load: true + tags: ${{ env.EMQX_IMAGE_TAG }} + build-args: | + EMQX_NAME=${{ env.EMQX_NAME }} + - name: test docker image + run: | + CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG) + HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID) + ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT + docker stop $CID + - name: export docker image + run: | + docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz + - uses: actions/upload-artifact@v3 + with: + name: "${{ matrix.profile }}-docker" + path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz" + spellcheck: needs: linux strategy: diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 6d4f53dbc..ff41a4e86 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 38e3a5b8b..393da4dbd 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index 892cd3a05..744618680 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -8,7 +8,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-latest # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 40d70a902..5f5450cab 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -7,7 +7,7 @@ on: [pull_request, push] jobs: elixir_deps_check: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index b93b3d63c..40bb83636 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,7 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-25.1.2-2-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-25.1.2-2-ubuntu20.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index df26778b4..fe21545ca 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -54,7 +54,7 @@ jobs: OUTPUT_DIR=${{ steps.profile.outputs.s3dir }} aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages cd packages - DEFAULT_BEAM_PLATFORM='otp24.3.4.2-1' + DEFAULT_BEAM_PLATFORM='otp24.3.4.2-2' # all packages including full-name and default-name are uploaded to s3 # but we only upload default-name packages (and elixir) as github artifacts # so we rename (overwrite) non-default packages before uploading diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 0b3f21b4a..cf6e1bdff 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,9 +12,9 @@ jobs: strategy: matrix: builder: - - 5.0-27 + - 5.0-28 otp: - - 24.3.4.2-1 + - 24.3.4.2-2 - 25.1.2-2 # no need to use more than 1 version of Elixir, since tests # run using only Erlang code. This is needed just to specify diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 994326d23..4ef634d91 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-debian11 + container: ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11 steps: - uses: actions/checkout@v3 @@ -49,9 +49,9 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-27 + - 5.0-28 otp: - - 24.3.4.2-1 + - 24.3.4.2-2 elixir: - 1.13.4 arch: @@ -122,9 +122,9 @@ jobs: os: - ["debian11", "debian:11-slim"] builder: - - 5.0-27 + - 5.0-28 otp: - - 24.3.4.2-1 + - 24.3.4.2-2 elixir: - 1.13.4 arch: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index d8d64d909..4d03878de 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,7 +15,7 @@ concurrency: jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 6b4357abc..79998f413 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -30,13 +30,13 @@ jobs: MATRIX="$(echo "${APPS}" | jq -c ' [ (.[] | select(.profile == "emqx") | . + { - builder: "5.0-27", + builder: "5.0-28", otp: "25.1.2-2", elixir: "1.13.4" }), (.[] | select(.profile == "emqx-enterprise") | . + { - builder: "5.0-27", - otp: ["24.3.4.2-1", "25.1.2-2"][], + builder: "5.0-28", + otp: ["24.3.4.2-2", "25.1.2-2"][], elixir: "1.13.4" }) ] @@ -56,7 +56,7 @@ jobs: echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT prepare: - runs-on: aws-amd64 + runs-on: ${{ needs.build-matrix.outputs.runs-on }} needs: [build-matrix] strategy: fail-fast: false @@ -161,6 +161,7 @@ jobs: PGSQL_TAG: "13" REDIS_TAG: "7.0" INFLUXDB_TAG: "2.5.0" + TDENGINE_TAG: "3.0.2.4" PROFILE: ${{ matrix.profile }} CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} @@ -223,12 +224,12 @@ jobs: - ct - ct_docker runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-27:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-emqx-enterprise-24.3.4.2-1 + name: source-emqx-enterprise-24.3.4.2-2 path: . - name: unzip source code run: unzip -q source.zip diff --git a/.tool-versions b/.tool-versions index 0f7c9b32e..dcf5945a8 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 24.3.4.2-1 +erlang 24.3.4.2-2 elixir 1.13.4-otp-24 diff --git a/Makefile b/Makefile index 2bfdffdfb..2cabaebcf 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,12 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) -export EMQX_DASHBOARD_VERSION ?= v1.1.7 -export EMQX_EE_DASHBOARD_VERSION ?= e1.0.3 +export EMQX_DASHBOARD_VERSION ?= v1.1.8 +export EMQX_EE_DASHBOARD_VERSION ?= e1.0.4-beta.3 export EMQX_REL_FORM ?= tgz export QUICER_DOWNLOAD_FROM_RELEASE = 1 ifeq ($(OS),Windows_NT) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 43dcfd411..ee345e9d6 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -34,6 +34,10 @@ listeners.wss.default { # enabled = true # bind = "0.0.0.0:14567" # max_connections = 1024000 -# keyfile = "{{ platform_etc_dir }}/certs/key.pem" -# certfile = "{{ platform_etc_dir }}/certs/cert.pem" -#} +# ssl_options { +# verify = verify_none +# keyfile = "{{ platform_etc_dir }}/certs/key.pem" +# certfile = "{{ platform_etc_dir }}/certs/cert.pem" +# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +# } +# } diff --git a/apps/emqx/i18n/emqx_schema_i18n.conf b/apps/emqx/i18n/emqx_schema_i18n.conf index 2ba5e7f52..39d5b2828 100644 --- a/apps/emqx/i18n/emqx_schema_i18n.conf +++ b/apps/emqx/i18n/emqx_schema_i18n.conf @@ -1495,6 +1495,17 @@ In case PSK cipher suites are intended, make sure to configure } } +common_ssl_opts_schema_hibernate_after { + desc { + en: """ Hibernate the SSL process after idling for amount of time reducing its memory footprint. """ + zh: """ 在闲置一定时间后休眠 SSL 进程，减少其内存占用。""" + } + label: { + en: "hibernate after" + zh: "闲置多久后休眠" + } +} + ciphers_schema_common { desc { en: """This config holds TLS cipher suite names separated by comma, @@ -1804,8 +1815,8 @@ fields_listener_enabled { fields_mqtt_quic_listener_certfile { desc { - en: """Path to the certificate file.""" - zh: """证书文件。""" + en: """Path to the certificate file. Will be deprecated in 5.1, use .ssl_options.certfile instead.""" + zh: """证书文件。在 5.1 中会被废弃，使用 .ssl_options.certfile 代替。""" } label: { en: "Certificate file" @@ -1815,8 +1826,8 @@ fields_mqtt_quic_listener_certfile { fields_mqtt_quic_listener_keyfile { desc { - en: """Path to the secret key file.""" - zh: """私钥文件。""" + en: """Path to the secret key file. Will be deprecated in 5.1, use .ssl_options.keyfile instead.""" + zh: """私钥文件。在 5.1 中会被废弃，使用 .ssl_options.keyfile 代替。""" } label: { en: "Key file" @@ -1857,6 +1868,17 @@ fields_mqtt_quic_listener_keep_alive_interval { } } +fields_mqtt_quic_listener_ssl_options { + desc { + en: """TLS options for QUIC transport""" + zh: """QUIC 传输层的 TLS 选项""" + } + label: { + en: "TLS Options" + zh: "TLS 选项" + } +} + base_listener_bind { desc { en: """IP address and port for the listening socket.""" diff --git a/apps/emqx/include/emqx_quic.hrl b/apps/emqx/include/emqx_quic.hrl new file mode 100644 index 000000000..a16784d5d --- /dev/null +++ b/apps/emqx/include/emqx_quic.hrl @@ -0,0 +1,25 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_QUIC_HRL). +-define(EMQX_QUIC_HRL, true). + +%% MQTT Over QUIC Shutdown Error code. +-define(MQTT_QUIC_CONN_NOERROR, 0). +-define(MQTT_QUIC_CONN_ERROR_CTRL_STREAM_DOWN, 1). +-define(MQTT_QUIC_CONN_ERROR_OVERLOADED, 2). + +-endif. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 7ea52a406..a3ea4f2e7 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -43,7 +43,7 @@ {meck, "0.9.2"}, {proper, "1.4.0"}, {bbmustache, "1.10.0"}, - {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0"}}} + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.2"}}} ]}, {extra_src_dirs, [{"test", [recursive]}]} ]} diff --git a/apps/emqx/rebar.config.script b/apps/emqx/rebar.config.script index 75f748017..2025f5ad5 100644 --- a/apps/emqx/rebar.config.script +++ b/apps/emqx/rebar.config.script @@ -24,7 +24,20 @@ IsQuicSupp = fun() -> end, Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}, -Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}. +Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.111"}}}. + +Dialyzer = fun(Config) -> + {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config), + {plt_extra_apps, OldExtra} = lists:keyfind(plt_extra_apps, 1, OldDialyzerConfig), + Extra = OldExtra ++ [quicer || IsQuicSupp()], + NewDialyzerConfig = [{plt_extra_apps, Extra} | OldDialyzerConfig], + lists:keystore( + dialyzer, + 1, + Config, + {dialyzer, NewDialyzerConfig} + ) + end. ExtraDeps = fun(C) -> {deps, Deps0} = lists:keyfind(deps, 1, C), @@ -43,4 +56,4 @@ ExtraDeps = fun(C) -> ) end, -ExtraDeps(CONFIG). +Dialyzer(ExtraDeps(CONFIG)). diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index c812b2217..3030ccb06 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -3,7 +3,7 @@ {id, "emqx"}, {description, "EMQX Core"}, % strict semver, bump manually! - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src index d3121c97b..04bf1f428 100644 --- a/apps/emqx/src/emqx.appup.src +++ b/apps/emqx/src/emqx.appup.src @@ -1,33 +1,5 @@ %% -*- mode: erlang -*- %% Unless you know what you are doing, DO NOT edit manually!! {VSN, - [{"5.0.0", - [{load_module,emqx_quic_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_config,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_schema,brutal_purge,soft_purge,[]}, - {load_module,emqx_release,brutal_purge,soft_purge,[]}, - {load_module,emqx_authentication,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {add_module,emqx_exclusive_subscription}, - {apply,{emqx_exclusive_subscription,on_add_module,[]}}, - {load_module,emqx_broker,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, - {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {load_module,emqx_relup}]}, - {<<".*">>,[]}], - [{"5.0.0", - [{load_module,emqx_quic_connection,brutal_purge,soft_purge,[]}, - {load_module,emqx_config,brutal_purge,soft_purge,[]}, - {load_module,emqx_channel,brutal_purge,soft_purge,[]}, - {load_module,emqx_schema,brutal_purge,soft_purge,[]}, - {load_module,emqx_release,brutal_purge,soft_purge,[]}, - {load_module,emqx_authentication,brutal_purge,soft_purge,[]}, - {load_module,emqx_metrics,brutal_purge,soft_purge,[]}, - {load_module,emqx_broker,brutal_purge,soft_purge,[]}, - {load_module,emqx_mqtt_caps,brutal_purge,soft_purge,[]}, - {load_module,emqx_topic,brutal_purge,soft_purge,[]}, - {apply,{emqx_exclusive_subscription,on_delete_module,[]}}, - {delete_module,emqx_exclusive_subscription}, - {load_module,emqx_relup}]}, - {<<".*">>,[]}]}. + [{<<".*">>,[]}], + [{<<".*">>,[]}]}. diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index 5b783f2fe..e5002cab4 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -14,7 +14,13 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% MQTT/TCP|TLS Connection +%% This module interacts with the transport layer of MQTT +%% Transport: +%% - TCP connection +%% - TCP/TLS connection +%% - QUIC Stream +%% +%% for WebSocket @see emqx_ws_connection.erl -module(emqx_connection). -include("emqx.hrl"). @@ -111,7 +117,10 @@ limiter_buffer :: queue:queue(pending_req()), %% limiter timers - limiter_timer :: undefined | reference() + limiter_timer :: undefined | reference(), + + %% QUIC conn owner pid if in use. + quic_conn_pid :: maybe(pid()) }). -record(retry, { @@ -189,12 +198,16 @@ ]} ). --spec start_link( - esockd:transport(), - esockd:socket() | {pid(), quicer:connection_handler()}, - emqx_channel:opts() -) -> - {ok, pid()}. +-spec start_link + (esockd:transport(), esockd:socket(), emqx_channel:opts()) -> + {ok, pid()}; + ( + emqx_quic_stream, + {ConnOwner :: pid(), quicer:connection_handle(), quicer:new_conn_props()}, + emqx_quic_connection:cb_state() + ) -> + {ok, pid()}. + start_link(Transport, Socket, Options) -> Args = [self(), Transport, Socket, Options], CPid = proc_lib:spawn_link(?MODULE, init, Args), @@ -329,6 +342,7 @@ init_state( }, ParseState = emqx_frame:initial_parse_state(FrameOpts), Serialize = emqx_frame:serialize_opts(), + %% Init Channel Channel = emqx_channel:init(ConnInfo, Opts), GcState = case emqx_config:get_zone_conf(Zone, [force_gc]) of @@ -359,7 +373,9 @@ init_state( zone = Zone, listener = Listener, limiter_buffer = queue:new(), - limiter_timer = undefined + limiter_timer = undefined, + %% for quic streams to inherit + quic_conn_pid = maps:get(conn_pid, Opts, undefined) }. run_loop( @@ -476,7 +492,9 @@ process_msg([Msg | More], State) -> {ok, Msgs, NState} -> process_msg(append_msg(More, Msgs), NState); {stop, Reason, NState} -> - {stop, Reason, NState} + {stop, Reason, NState}; + {stop, Reason} -> + {stop, Reason, State} end catch exit:normal -> @@ -507,7 +525,6 @@ append_msg(Q, Msg) -> %%-------------------------------------------------------------------- %% Handle a Msg - handle_msg({'$gen_call', From, Req}, State) -> case handle_call(From, Req, State) of {reply, Reply, NState} -> @@ -525,11 +542,10 @@ handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl -> inc_counter(incoming_bytes, Oct), ok = emqx_metrics:inc('bytes.received', Oct), when_bytes_in(Oct, Data, State); -handle_msg({quic, Data, _Sock, _, _, _}, State) -> - Oct = iolist_size(Data), - inc_counter(incoming_bytes, Oct), - ok = emqx_metrics:inc('bytes.received', Oct), - when_bytes_in(Oct, Data, State); +handle_msg({quic, Data, _Stream, #{len := Len}}, State) when is_binary(Data) -> + inc_counter(incoming_bytes, Len), + ok = emqx_metrics:inc('bytes.received', Len), + when_bytes_in(Len, Data, State); handle_msg(check_cache, #state{limiter_buffer = Cache} = State) -> case queue:peek(Cache) of empty -> @@ -595,9 +611,20 @@ handle_msg({inet_reply, _Sock, {error, Reason}}, State) -> handle_msg({connack, ConnAck}, State) -> handle_outgoing(ConnAck, State); handle_msg({close, Reason}, State) -> + %% @FIXME here it could be close due to appl error. ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}), handle_info({sock_closed, Reason}, close_socket(State)); -handle_msg({event, connected}, State = #state{channel = Channel}) -> +handle_msg( + {event, connected}, + State = #state{ + channel = Channel, + serialize = Serialize, + parse_state = PS, + quic_conn_pid = QuicConnPid + } +) -> + QuicConnPid =/= undefined andalso + emqx_quic_connection:activate_data_streams(QuicConnPid, {PS, Serialize, Channel}), ClientId = emqx_channel:info(clientid, Channel), emqx_cm:insert_channel_info(ClientId, info(State), stats(State)); handle_msg({event, disconnected}, State = #state{channel = Channel}) -> @@ -654,6 +681,12 @@ maybe_raise_exception(#{ stacktrace := Stacktrace }) -> erlang:raise(Exception, Context, Stacktrace); +maybe_raise_exception({shutdown, normal}) -> + ok; +maybe_raise_exception(normal) -> + ok; +maybe_raise_exception(shutdown) -> + ok; maybe_raise_exception(Reason) -> exit(Reason). @@ -748,6 +781,7 @@ when_bytes_in(Oct, Data, State) -> NState ). +%% @doc: return a reversed Msg list -compile({inline, [next_incoming_msgs/3]}). next_incoming_msgs([Packet], Msgs, State) -> {ok, [{incoming, Packet} | Msgs], State}; @@ -870,6 +904,7 @@ send(IoData, #state{transport = Transport, socket = Socket, channel = Channel}) ok; Error = {error, _Reason} -> %% Send an inet_reply to postpone handling the error + %% @FIXME: why not just return error? self() ! {inet_reply, Socket, Error}, ok end. @@ -893,12 +928,14 @@ handle_info({sock_error, Reason}, State) -> false -> ok end, handle_info({sock_closed, Reason}, close_socket(State)); -handle_info({quic, peer_send_shutdown, _Stream}, State) -> - handle_info({sock_closed, force}, close_socket(State)); -handle_info({quic, closed, _Channel, ReasonFlag}, State) -> - handle_info({sock_closed, ReasonFlag}, State); -handle_info({quic, closed, _Stream}, State) -> - handle_info({sock_closed, force}, State); +%% handle QUIC control stream events +handle_info({quic, Event, Handle, Prop}, State) when is_atom(Event) -> + case emqx_quic_stream:Event(Handle, Prop, State) of + {{continue, Msgs}, NewState} -> + {ok, Msgs, NewState}; + Other -> + Other + end; handle_info(Info, State) -> with_channel(handle_info, [Info], State). diff --git a/apps/emqx/src/emqx_kernel_sup.erl b/apps/emqx/src/emqx_kernel_sup.erl index 21ed8576a..a69674de8 100644 --- a/apps/emqx/src/emqx_kernel_sup.erl +++ b/apps/emqx/src/emqx_kernel_sup.erl @@ -35,7 +35,6 @@ init([]) -> child_spec(emqx_hooks, worker), child_spec(emqx_stats, worker), child_spec(emqx_metrics, worker), - child_spec(emqx_ctl, worker), child_spec(emqx_authn_authz_metrics_sup, supervisor) ] }}. diff --git a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl index fa67e1977..ddfc55f7a 100644 --- a/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl +++ b/apps/emqx/src/emqx_limiter/src/emqx_limiter_schema.erl @@ -110,11 +110,11 @@ fields(limiter) -> ]; fields(node_opts) -> [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, + {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, {burst, ?HOCON(burst_rate(), #{ desc => ?DESC(burst), - default => 0 + default => <<"0">> })} ]; fields(client_fields) -> @@ -128,14 +128,14 @@ fields(client_fields) -> ]; fields(bucket_opts) -> [ - {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => "infinity"})}, - {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => "infinity"})}, - {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})} + {rate, ?HOCON(rate(), #{desc => ?DESC(rate), default => <<"infinity">>})}, + {capacity, ?HOCON(capacity(), #{desc => ?DESC(capacity), default => <<"infinity">>})}, + {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})} ]; fields(client_opts) -> [ - {rate, ?HOCON(rate(), #{default => "infinity", desc => ?DESC(rate)})}, - {initial, ?HOCON(initial(), #{default => "0", desc => ?DESC(initial)})}, + {rate, ?HOCON(rate(), #{default => <<"infinity">>, desc => ?DESC(rate)})}, + {initial, ?HOCON(initial(), #{default => <<"0">>, desc => ?DESC(initial)})}, %% low_watermark add for emqx_channel and emqx_session %% both modules consume first and then check %% so we need to use this value to prevent excessive consumption @@ -145,13 +145,13 @@ fields(client_opts) -> initial(), #{ desc => ?DESC(low_watermark), - default => "0" + default => <<"0">> } )}, {capacity, ?HOCON(capacity(), #{ desc => ?DESC(client_bucket_capacity), - default => "infinity" + default => <<"infinity">> })}, {divisible, ?HOCON( @@ -166,7 +166,7 @@ fields(client_opts) -> emqx_schema:duration(), #{ desc => ?DESC(max_retry_time), - default => "10s" + default => <<"10s">> } )}, {failure_strategy, diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 003c8785e..fedf583e2 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -72,9 +72,7 @@ id_example() -> 'tcp:default'. list_raw() -> [ {listener_id(Type, LName), Type, LConf} - || %% FIXME: quic is not supported update vi dashboard yet - {Type, LName, LConf} <- do_list_raw(), - Type =/= <<"quic">> + || {Type, LName, LConf} <- do_list_raw() ]. list() -> @@ -170,6 +168,11 @@ current_conns(Type, Name, ListenOn) when Type == tcp; Type == ssl -> esockd:get_current_connections({listener_id(Type, Name), ListenOn}); current_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> proplists:get_value(all_connections, ranch:info(listener_id(Type, Name))); +current_conns(quic, _Name, _ListenOn) -> + case quicer:perf_counters() of + {ok, PerfCnts} -> proplists:get_value(conn_active, PerfCnts); + _ -> 0 + end; current_conns(_, _, _) -> {error, not_support}. @@ -367,16 +370,26 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> case [A || {quicer, _, _} = A <- application:which_applications()] of [_] -> DefAcceptors = erlang:system_info(schedulers_online) * 8, - ListenOpts = [ - {cert, maps:get(certfile, Opts)}, - {key, maps:get(keyfile, Opts)}, - {alpn, ["mqtt"]}, - {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])}, - {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)}, - {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)}, - {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)}, - {server_resumption_level, 2} - ], + SSLOpts = maps:merge( + maps:with([certfile, keyfile], Opts), + maps:get(ssl_options, Opts, #{}) + ), + ListenOpts = + [ + {certfile, str(maps:get(certfile, SSLOpts))}, + {keyfile, str(maps:get(keyfile, SSLOpts))}, + {alpn, ["mqtt"]}, + {conn_acceptors, lists:max([DefAcceptors, maps:get(acceptors, Opts, 0)])}, + {keep_alive_interval_ms, maps:get(keep_alive_interval, Opts, 0)}, + {idle_timeout_ms, maps:get(idle_timeout, Opts, 0)}, + {handshake_idle_timeout_ms, maps:get(handshake_idle_timeout, Opts, 10000)}, + {server_resumption_level, 2}, + {verify, maps:get(verify, SSLOpts, verify_none)} + ] ++ + case maps:get(cacertfile, SSLOpts, undefined) of + undefined -> []; + CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}] + end, ConnectionOpts = #{ conn_callback => emqx_quic_connection, peer_unidi_stream_count => 1, @@ -385,13 +398,16 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) -> listener => {quic, ListenerName}, limiter => limiter(Opts) }, - StreamOpts = [{stream_callback, emqx_quic_stream}], + StreamOpts = #{ + stream_callback => emqx_quic_stream, + active => 1 + }, Id = listener_id(quic, ListenerName), add_limiter_bucket(Id, Opts), quicer:start_listener( Id, ListenOn, - {ListenOpts, ConnectionOpts, StreamOpts} + {maps:from_list(ListenOpts), ConnectionOpts, StreamOpts} ); [] -> {ok, {skipped, quic_app_missing}} diff --git a/apps/emqx/src/emqx_quic_connection.erl b/apps/emqx/src/emqx_quic_connection.erl index 9a2589a3a..a77ec28f2 100644 --- a/apps/emqx/src/emqx_quic_connection.erl +++ b/apps/emqx/src/emqx_quic_connection.erl @@ -14,60 +14,282 @@ %% limitations under the License. %%-------------------------------------------------------------------- +%% @doc impl. the quic connection owner process. -module(emqx_quic_connection). -ifndef(BUILD_WITHOUT_QUIC). --include_lib("quicer/include/quicer.hrl"). --else. --define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0). --endif. -%% Callbacks +-include("logger.hrl"). +-include_lib("quicer/include/quicer.hrl"). +-include_lib("emqx/include/emqx_quic.hrl"). + +-behaviour(quicer_connection). + -export([ init/1, - new_conn/2, - connected/2, - shutdown/2 + new_conn/3, + connected/3, + transport_shutdown/3, + shutdown/3, + closed/3, + local_address_changed/3, + peer_address_changed/3, + streams_available/3, + peer_needs_streams/3, + resumed/3, + new_stream/3 ]). --type cb_state() :: map() | proplists:proplist(). +-export([activate_data_streams/2]). --spec init(cb_state()) -> cb_state(). -init(ConnOpts) when is_list(ConnOpts) -> - init(maps:from_list(ConnOpts)); +-export([ + handle_call/3, + handle_info/2 +]). + +-type cb_state() :: #{ + %% connecion owner pid + conn_pid := pid(), + %% Pid of ctrl stream + ctrl_pid := undefined | pid(), + %% quic connecion handle + conn := undefined | quicer:conneciton_handle(), + %% Data streams that handoff from this process + %% these streams could die/close without effecting the connecion/session. + %@TODO type? + streams := [{pid(), quicer:stream_handle()}], + %% New stream opts + stream_opts := map(), + %% If conneciton is resumed from session ticket + is_resumed => boolean(), + %% mqtt message serializer config + serialize => undefined, + _ => _ +}. +-type cb_ret() :: quicer_lib:cb_ret(). + +%% @doc Data streams initializions are started in parallel with control streams, data streams are blocked +%% for the activation from control stream after it is accepted as a legit conneciton. +%% For security, the initial number of allowed data streams from client should be limited by +%% 'peer_bidi_stream_count` & 'peer_unidi_stream_count` +-spec activate_data_streams(pid(), { + emqx_frame:parse_state(), emqx_frame:serialize_opts(), emqx_channel:channel() +}) -> ok. +activate_data_streams(ConnOwner, {PS, Serialize, Channel}) -> + gen_server:call(ConnOwner, {activate_data_streams, {PS, Serialize, Channel}}, infinity). + +%% @doc conneciton owner init callback +-spec init(map()) -> {ok, cb_state()}. +init(#{stream_opts := SOpts} = S) when is_list(SOpts) -> + init(S#{stream_opts := maps:from_list(SOpts)}); init(ConnOpts) when is_map(ConnOpts) -> - ConnOpts. + {ok, init_cb_state(ConnOpts)}. --spec new_conn(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. -new_conn(Conn, #{zone := Zone} = S) -> +-spec closed(quicer:conneciton_handle(), quicer:conn_closed_props(), cb_state()) -> + {stop, normal, cb_state()}. +closed(_Conn, #{is_peer_acked := _} = Prop, S) -> + ?SLOG(debug, Prop), + {stop, normal, S}. + +%% @doc handle the new incoming connecion as the connecion acceptor. +-spec new_conn(quicer:connection_handle(), quicer:new_conn_props(), cb_state()) -> + {ok, cb_state()} | {error, any(), cb_state()}. +new_conn( + Conn, + #{version := _Vsn} = ConnInfo, + #{zone := Zone, conn := undefined, ctrl_pid := undefined} = S +) -> process_flag(trap_exit, true), + ?SLOG(debug, ConnInfo), case emqx_olp:is_overloaded() andalso is_zone_olp_enabled(Zone) of false -> - {ok, Pid} = emqx_connection:start_link(emqx_quic_stream, {self(), Conn}, S), + %% Start control stream process + StartOption = S, + {ok, CtrlPid} = emqx_connection:start_link( + emqx_quic_stream, + {self(), Conn, maps:without([crypto_buffer], ConnInfo)}, + StartOption + ), receive - {Pid, stream_acceptor_ready} -> + {CtrlPid, stream_acceptor_ready} -> ok = quicer:async_handshake(Conn), - {ok, S}; - {'EXIT', Pid, _Reason} -> - {error, stream_accept_error} + {ok, S#{conn := Conn, ctrl_pid := CtrlPid}}; + {'EXIT', _Pid, _Reason} -> + {stop, stream_accept_error, S} end; true -> emqx_metrics:inc('olp.new_conn'), - {error, overloaded} + _ = quicer:async_shutdown_connection( + Conn, + ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, + ?MQTT_QUIC_CONN_ERROR_OVERLOADED + ), + {stop, normal, S} end. --spec connected(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. -connected(Conn, #{slow_start := false} = S) -> - {ok, _Pid} = emqx_connection:start_link(emqx_quic_stream, Conn, S), +%% @doc callback when connection is connected. +-spec connected(quicer:connection_handle(), quicer:connected_props(), cb_state()) -> + {ok, cb_state()} | {error, any(), cb_state()}. +connected(_Conn, Props, S) -> + ?SLOG(debug, Props), + {ok, S}. + +%% @doc callback when connection is resumed from 0-RTT +-spec resumed(quicer:connection_handle(), SessionData :: binary() | false, cb_state()) -> cb_ret(). +%% reserve resume conn with callback. +%% resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when +%% is_function(ResumeFun) +%% -> +%% ResumeFun(Conn, Data, S); +resumed(_Conn, _Data, S) -> + {ok, S#{is_resumed := true}}. + +%% @doc callback for handling orphan data streams +%% depends on the connecion state and control stream state. +-spec new_stream(quicer:stream_handle(), quicer:new_stream_props(), cb_state()) -> cb_ret(). +new_stream( + Stream, + #{is_orphan := true, flags := _Flags} = Props, + #{ + conn := Conn, + streams := Streams, + stream_opts := SOpts, + zone := Zone, + limiter := Limiter, + parse_state := PS, + channel := Channel, + serialize := Serialize + } = S +) -> + %% Cherry pick options for data streams + SOpts1 = SOpts#{ + is_local => false, + zone => Zone, + % unused + limiter => Limiter, + parse_state => PS, + channel => Channel, + serialize => Serialize, + quic_event_mask => ?QUICER_STREAM_EVENT_MASK_START_COMPLETE + }, + {ok, NewStreamOwner} = quicer_stream:start_link( + emqx_quic_data_stream, + Stream, + Conn, + SOpts1, + Props + ), + case quicer:handoff_stream(Stream, NewStreamOwner, {PS, Serialize, Channel}) of + ok -> + ok; + E -> + %% Only log, keep connecion alive. + ?SLOG(error, #{message => "new stream handoff failed", stream => Stream, error => E}) + end, + %% @TODO maybe keep them in `inactive_streams' + {ok, S#{streams := [{NewStreamOwner, Stream} | Streams]}}. + +%% @doc callback for handling remote connecion shutdown. +-spec shutdown(quicer:connection_handle(), quicer:error_code(), cb_state()) -> cb_ret(). +shutdown(Conn, ErrorCode, S) -> + ErrorCode =/= 0 andalso ?SLOG(debug, #{error_code => ErrorCode, state => S}), + _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), + {ok, S}. + +%% @doc callback for handling transport error, such as idle timeout +-spec transport_shutdown(quicer:connection_handle(), quicer:transport_shutdown_props(), cb_state()) -> + cb_ret(). +transport_shutdown(_C, DownInfo, S) when is_map(DownInfo) -> + ?SLOG(debug, DownInfo), + {ok, S}. + +%% @doc callback for handling for peer addr changed. +-spec peer_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state) -> cb_ret(). +peer_address_changed(_C, _NewAddr, S) -> + %% @TODO update conn info in emqx_quic_stream + {ok, S}. + +%% @doc callback for handling local addr change, currently unused +-spec local_address_changed(quicer:connection_handle(), quicer:quicer_addr(), cb_state()) -> + cb_ret(). +local_address_changed(_C, _NewAddr, S) -> + {ok, S}. + +%% @doc callback for handling remote stream limit updates +-spec streams_available( + quicer:connection_handle(), + {BidirStreams :: non_neg_integer(), UnidirStreams :: non_neg_integer()}, + cb_state() +) -> cb_ret(). +streams_available(_C, {BidirCnt, UnidirCnt}, S) -> + {ok, S#{ + peer_bidi_stream_count => BidirCnt, + peer_unidi_stream_count => UnidirCnt + }}. + +%% @doc callback for handling request when remote wants for more streams +%% should cope with rate limiting +%% @TODO this is not going to get triggered in current version +%% ref: https://github.com/microsoft/msquic/issues/3120 +-spec peer_needs_streams(quicer:connection_handle(), undefined, cb_state()) -> cb_ret(). +peer_needs_streams(_C, undefined, S) -> + ?SLOG(info, #{ + msg => "ignore: peer need more streames", info => maps:with([conn_pid, ctrl_pid], S) + }), + {ok, S}. + +%% @doc handle API calls +-spec handle_call(Req :: term(), gen_server:from(), cb_state()) -> cb_ret(). +handle_call( + {activate_data_streams, {PS, Serialize, Channel} = ActivateData}, + _From, + #{streams := Streams} = S +) -> + _ = [ + %% Try to activate streams individually if failed, stream will shutdown on its own. + %% we dont care about the return val here. + %% note, this is only used after control stream pass the validation. The data streams + %% that are called here are assured to be inactived (data processing hasn't been started). + catch emqx_quic_data_stream:activate_data(OwnerPid, ActivateData) + || {OwnerPid, _Stream} <- Streams + ], + {reply, ok, S#{ + channel := Channel, + serialize := Serialize, + parse_state := PS + }}; +handle_call(_Req, _From, S) -> + {reply, {error, unimpl}, S}. + +%% @doc handle DOWN messages from streams. +handle_info({'EXIT', Pid, Reason}, #{ctrl_pid := Pid, conn := Conn} = S) -> + Code = + case Reason of + normal -> + ?MQTT_QUIC_CONN_NOERROR; + _ -> + ?MQTT_QUIC_CONN_ERROR_CTRL_STREAM_DOWN + end, + _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, Code), {ok, S}; -connected(_Conn, S) -> - {ok, S}. - --spec shutdown(quicer:connection_handler(), cb_state()) -> {ok, cb_state()} | {error, any()}. -shutdown(Conn, S) -> - quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), - {ok, S}. +handle_info({'EXIT', Pid, Reason}, #{streams := Streams} = S) -> + case proplists:is_defined(Pid, Streams) of + true when + Reason =:= normal orelse + Reason =:= {shutdown, protocol_error} orelse + Reason =:= killed + -> + {ok, S}; + true -> + ?SLOG(info, #{message => "Data stream unexpected exit", reason => Reason}), + {ok, S}; + false -> + {stop, unknown_pid_down, S} + end. +%%% +%%% Internals +%%% -spec is_zone_olp_enabled(emqx_types:zone()) -> boolean(). is_zone_olp_enabled(Zone) -> case emqx_config:get_zone_conf(Zone, [overload_protection]) of @@ -76,3 +298,20 @@ is_zone_olp_enabled(Zone) -> _ -> false end. + +-spec init_cb_state(map()) -> cb_state(). +init_cb_state(#{zone := _Zone} = Map) -> + Map#{ + conn_pid => self(), + ctrl_pid => undefined, + conn => undefined, + streams => [], + parse_state => undefined, + channel => undefined, + serialize => undefined, + is_resumed => false + }. + +%% BUILD_WITHOUT_QUIC +-else. +-endif. diff --git a/apps/emqx/src/emqx_quic_data_stream.erl b/apps/emqx/src/emqx_quic_data_stream.erl new file mode 100644 index 000000000..0b89870a8 --- /dev/null +++ b/apps/emqx/src/emqx_quic_data_stream.erl @@ -0,0 +1,469 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% +%% @doc QUIC data stream +%% Following the behaviour of emqx_connection: +%% The MQTT packets and their side effects are handled *atomically*. +%% + +-module(emqx_quic_data_stream). + +-ifndef(BUILD_WITHOUT_QUIC). +-behaviour(quicer_remote_stream). + +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("quicer/include/quicer.hrl"). +-include("emqx_mqtt.hrl"). +-include("logger.hrl"). + +%% Connection Callbacks +-export([ + init_handoff/4, + post_handoff/3, + send_complete/3, + peer_send_shutdown/3, + peer_send_aborted/3, + peer_receive_aborted/3, + send_shutdown_complete/3, + stream_closed/3, + passive/3 +]). + +-export([handle_stream_data/4]). + +%% gen_server API +-export([activate_data/2]). + +-export([ + handle_call/3, + handle_info/2, + handle_continue/2 +]). + +-type cb_ret() :: quicer_stream:cb_ret(). +-type cb_state() :: quicer_stream:cb_state(). +-type error_code() :: quicer:error_code(). +-type connection_handle() :: quicer:connection_handle(). +-type stream_handle() :: quicer:stream_handle(). +-type handoff_data() :: { + emqx_frame:parse_state() | undefined, + emqx_frame:serialize_opts() | undefined, + emqx_channel:channel() | undefined +}. +%% +%% @doc Activate the data handling. +%% Note, data handling is disabled before finishing the validation over control stream. +-spec activate_data(pid(), { + emqx_frame:parse_state(), emqx_frame:serialize_opts(), emqx_channel:channel() +}) -> ok. +activate_data(StreamPid, {PS, Serialize, Channel}) -> + gen_server:call(StreamPid, {activate, {PS, Serialize, Channel}}, infinity). + +%% +%% @doc Handoff from previous owner, from the connection owner. +%% Note, unlike control stream, there is no acceptor for data streams. +%% The connection owner get new stream, spawn new proc and then handover to it. +%% +-spec init_handoff(stream_handle(), map(), connection_handle(), quicer:new_stream_props()) -> + {ok, cb_state()}. +init_handoff( + Stream, + _StreamOpts, + Connection, + #{is_orphan := true, flags := Flags} +) -> + {ok, init_state(Stream, Connection, Flags)}. + +%% +%% @doc Post handoff data stream +%% +-spec post_handoff(stream_handle(), handoff_data(), cb_state()) -> cb_ret(). +post_handoff(_Stream, {undefined = _PS, undefined = _Serialize, undefined = _Channel}, S) -> + %% When the channel isn't ready yet. + %% Data stream should wait for activate call with ?MODULE:activate_data/2 + {ok, S}; +post_handoff(Stream, {PS, Serialize, Channel}, S) -> + ?tp(debug, ?FUNCTION_NAME, #{channel => Channel, serialize => Serialize}), + _ = quicer:setopt(Stream, active, 10), + {ok, S#{channel := Channel, serialize := Serialize, parse_state := PS}}. + +-spec peer_receive_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret(). +peer_receive_aborted(Stream, ErrorCode, #{is_unidir := _} = S) -> + %% we abort send with same reason + _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + {ok, S}. + +-spec peer_send_aborted(stream_handle(), error_code(), cb_state()) -> cb_ret(). +peer_send_aborted(Stream, ErrorCode, #{is_unidir := _} = S) -> + %% we abort receive with same reason + _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, ErrorCode), + {ok, S}. + +-spec peer_send_shutdown(stream_handle(), undefined, cb_state()) -> cb_ret(). +peer_send_shutdown(Stream, undefined, S) -> + ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0), + {ok, S}. + +-spec send_complete(stream_handle(), IsCanceled :: boolean(), cb_state()) -> cb_ret(). +send_complete(_Stream, false, S) -> + {ok, S}; +send_complete(_Stream, true = _IsCanceled, S) -> + {ok, S}. + +-spec send_shutdown_complete(stream_handle(), error_code(), cb_state()) -> cb_ret(). +send_shutdown_complete(_Stream, _Flags, S) -> + {ok, S}. + +-spec handle_stream_data(stream_handle(), binary(), quicer:recv_data_props(), cb_state()) -> + cb_ret(). +handle_stream_data( + _Stream, + Bin, + _Flags, + #{ + is_unidir := false, + channel := Channel, + parse_state := PS, + data_queue := QueuedData, + task_queue := TQ + } = State +) when + %% assert get stream data only after channel is created + Channel =/= undefined +-> + {MQTTPackets, NewPS} = parse_incoming(list_to_binary(lists:reverse([Bin | QueuedData])), PS), + NewTQ = lists:foldl( + fun(Item, Acc) -> + queue:in(Item, Acc) + end, + TQ, + [{incoming, P} || P <- lists:reverse(MQTTPackets)] + ), + {{continue, handle_appl_msg}, State#{parse_state := NewPS, task_queue := NewTQ}}. + +-spec passive(stream_handle(), undefined, cb_state()) -> cb_ret(). +passive(Stream, undefined, S) -> + _ = quicer:setopt(Stream, active, 10), + {ok, S}. + +-spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_state()) -> cb_ret(). +stream_closed( + _Stream, + #{ + is_conn_shutdown := IsConnShutdown, + is_app_closing := IsAppClosing, + is_shutdown_by_app := IsAppShutdown, + is_closed_remotely := IsRemote, + status := Status, + error := Code + }, + S +) when + is_boolean(IsConnShutdown) andalso + is_boolean(IsAppClosing) andalso + is_boolean(IsAppShutdown) andalso + is_boolean(IsRemote) andalso + is_atom(Status) andalso + is_integer(Code) +-> + {stop, normal, S}. + +-spec handle_call(Request :: term(), From :: {pid(), term()}, cb_state()) -> cb_ret(). +handle_call(Call, _From, S) -> + do_handle_call(Call, S). + +-spec handle_continue(Continue :: term(), cb_state()) -> cb_ret(). +handle_continue(handle_appl_msg, #{task_queue := Q} = S) -> + case queue:out(Q) of + {{value, Item}, Q2} -> + do_handle_appl_msg(Item, S#{task_queue := Q2}); + {empty, _Q} -> + {ok, S} + end. + +%%% Internals +do_handle_appl_msg( + {outgoing, Packets}, + #{ + channel := Channel, + stream := _Stream, + serialize := _Serialize + } = S +) when + Channel =/= undefined +-> + case handle_outgoing(Packets, S) of + {ok, Size} -> + ok = emqx_metrics:inc('bytes.sent', Size), + {{continue, handle_appl_msg}, S}; + {error, E1, E2} -> + {stop, {E1, E2}, S}; + {error, E} -> + {stop, E, S} + end; +do_handle_appl_msg({incoming, #mqtt_packet{} = Packet}, #{channel := Channel} = S) when + Channel =/= undefined +-> + ok = inc_incoming_stats(Packet), + with_channel(handle_in, [Packet], S); +do_handle_appl_msg({incoming, {frame_error, _} = FE}, #{channel := Channel} = S) when + Channel =/= undefined +-> + with_channel(handle_in, [FE], S); +do_handle_appl_msg({close, Reason}, S) -> + %% @TODO shall we abort shutdown or graceful shutdown here? + with_channel(handle_info, [{sock_closed, Reason}], S); +do_handle_appl_msg({event, updated}, S) -> + %% Data stream don't care about connection state changes. + {{continue, handle_appl_msg}, S}. + +handle_info(Deliver = {deliver, _, _}, S) -> + Delivers = [Deliver], + with_channel(handle_deliver, [Delivers], S); +handle_info({timeout, Ref, Msg}, S) -> + with_channel(handle_timeout, [Ref, Msg], S); +handle_info(Info, State) -> + with_channel(handle_info, [Info], State). + +with_channel(Fun, Args, #{channel := Channel, task_queue := Q} = S) when + Channel =/= undefined +-> + case apply(emqx_channel, Fun, Args ++ [Channel]) of + ok -> + {{continue, handle_appl_msg}, S}; + {ok, Msgs, NewChannel} when is_list(Msgs) -> + {{continue, handle_appl_msg}, S#{ + task_queue := queue:join(Q, queue:from_list(Msgs)), + channel := NewChannel + }}; + {ok, Msg, NewChannel} when is_record(Msg, mqtt_packet) -> + {{continue, handle_appl_msg}, S#{ + task_queue := queue:in({outgoing, Msg}, Q), channel := NewChannel + }}; + %% @FIXME WTH? + {ok, {outgoing, _} = Msg, NewChannel} -> + {{continue, handle_appl_msg}, S#{task_queue := queue:in(Msg, Q), channel := NewChannel}}; + {ok, NewChannel} -> + {{continue, handle_appl_msg}, S#{channel := NewChannel}}; + %% @TODO optimisation for shutdown wrap + {shutdown, Reason, NewChannel} -> + {stop, {shutdown, Reason}, S#{channel := NewChannel}}; + {shutdown, Reason, Msgs, NewChannel} when is_list(Msgs) -> + %% @TODO handle outgoing? + {stop, {shutdown, Reason}, S#{ + channel := NewChannel, + task_queue := queue:join(Q, queue:from_list(Msgs)) + }}; + {shutdown, Reason, Msg, NewChannel} -> + {stop, {shutdown, Reason}, S#{ + channel := NewChannel, + task_queue := queue:in(Msg, Q) + }} + end. + +handle_outgoing(#mqtt_packet{} = P, S) -> + handle_outgoing([P], S); +handle_outgoing(Packets, #{serialize := Serialize, stream := Stream, is_unidir := false}) when + is_list(Packets) +-> + OutBin = [serialize_packet(P, Serialize) || P <- filter_disallowed_out(Packets)], + %% Send data async but still want send feedback via {quic, send_complete, ...} + Res = quicer:async_send(Stream, OutBin, ?QUICER_SEND_FLAG_SYNC), + ?TRACE("MQTT", "mqtt_packet_sent", #{packets => Packets}), + [ok = inc_outgoing_stats(P) || P <- Packets], + Res. + +serialize_packet(Packet, Serialize) -> + try emqx_frame:serialize_pkt(Packet, Serialize) of + <<>> -> + ?SLOG(warning, #{ + msg => "packet_is_discarded", + reason => "frame_is_too_large", + packet => emqx_packet:format(Packet, hidden) + }), + ok = emqx_metrics:inc('delivery.dropped.too_large'), + ok = emqx_metrics:inc('delivery.dropped'), + ok = inc_outgoing_stats({error, message_too_large}), + <<>>; + Data -> + Data + catch + %% Maybe Never happen. + throw:{?FRAME_SERIALIZE_ERROR, Reason} -> + ?SLOG(info, #{ + reason => Reason, + input_packet => Packet + }), + erlang:error({?FRAME_SERIALIZE_ERROR, Reason}); + error:Reason:Stacktrace -> + ?SLOG(error, #{ + input_packet => Packet, + exception => Reason, + stacktrace => Stacktrace + }), + erlang:error(?FRAME_SERIALIZE_ERROR) + end. + +-spec init_state( + quicer:stream_handle(), + quicer:connection_handle(), + quicer:new_stream_props() +) -> + % @TODO + map(). +init_state(Stream, Connection, OpenFlags) -> + init_state(Stream, Connection, OpenFlags, undefined). + +init_state(Stream, Connection, OpenFlags, PS) -> + %% quic stream handle + #{ + stream => Stream, + %% quic connection handle + conn => Connection, + %% if it is QUIC unidi stream + is_unidir => quicer:is_unidirectional(OpenFlags), + %% Frame Parse State + parse_state => PS, + %% Peer Stream handle in a pair for type unidir only + peer_stream => undefined, + %% if the stream is locally initiated. + is_local => false, + %% queue binary data when is NOT connected, in reversed order. + data_queue => [], + %% Channel from connection + %% `undefined' means the connection is not connected. + channel => undefined, + %% serialize opts for connection + serialize => undefined, + %% Current working queue + task_queue => queue:new() + }. + +-spec do_handle_call(term(), cb_state()) -> cb_ret(). +do_handle_call( + {activate, {PS, Serialize, Channel}}, + #{ + channel := undefined, + stream := Stream, + serialize := undefined + } = S +) -> + NewS = S#{channel := Channel, serialize := Serialize, parse_state := PS}, + %% We use quic protocol for flow control, and we don't check return val + case quicer:setopt(Stream, active, true) of + ok -> + {reply, ok, NewS}; + {error, E} -> + ?SLOG(error, #{msg => "set stream active failed", error => E}), + {stop, E, NewS} + end; +do_handle_call(_Call, _S) -> + {error, unimpl}. + +%% @doc return reserved order of Packets +parse_incoming(Data, PS) -> + try + do_parse_incoming(Data, [], PS) + catch + throw:{?FRAME_PARSE_ERROR, Reason} -> + ?SLOG(info, #{ + reason => Reason, + input_bytes => Data + }), + {[{frame_error, Reason}], PS}; + error:Reason:Stacktrace -> + ?SLOG(error, #{ + input_bytes => Data, + reason => Reason, + stacktrace => Stacktrace + }), + {[{frame_error, Reason}], PS} + end. + +do_parse_incoming(<<>>, Packets, ParseState) -> + {Packets, ParseState}; +do_parse_incoming(Data, Packets, ParseState) -> + case emqx_frame:parse(Data, ParseState) of + {more, NParseState} -> + {Packets, NParseState}; + {ok, Packet, Rest, NParseState} -> + do_parse_incoming(Rest, [Packet | Packets], NParseState) + end. + +%% followings are copied from emqx_connection +-compile({inline, [inc_incoming_stats/1]}). +inc_incoming_stats(Packet = ?PACKET(Type)) -> + inc_counter(recv_pkt, 1), + case Type =:= ?PUBLISH of + true -> + inc_counter(recv_msg, 1), + inc_qos_stats(recv_msg, Packet), + inc_counter(incoming_pubs, 1); + false -> + ok + end, + emqx_metrics:inc_recv(Packet). + +-compile({inline, [inc_outgoing_stats/1]}). +inc_outgoing_stats({error, message_too_large}) -> + inc_counter('send_msg.dropped', 1), + inc_counter('send_msg.dropped.too_large', 1); +inc_outgoing_stats(Packet = ?PACKET(Type)) -> + inc_counter(send_pkt, 1), + case Type of + ?PUBLISH -> + inc_counter(send_msg, 1), + inc_counter(outgoing_pubs, 1), + inc_qos_stats(send_msg, Packet); + _ -> + ok + end, + emqx_metrics:inc_sent(Packet). + +inc_counter(Key, Inc) -> + _ = emqx_pd:inc_counter(Key, Inc), + ok. + +inc_qos_stats(Type, Packet) -> + case inc_qos_stats_key(Type, emqx_packet:qos(Packet)) of + undefined -> + ignore; + Key -> + inc_counter(Key, 1) + end. + +inc_qos_stats_key(send_msg, ?QOS_0) -> 'send_msg.qos0'; +inc_qos_stats_key(send_msg, ?QOS_1) -> 'send_msg.qos1'; +inc_qos_stats_key(send_msg, ?QOS_2) -> 'send_msg.qos2'; +inc_qos_stats_key(recv_msg, ?QOS_0) -> 'recv_msg.qos0'; +inc_qos_stats_key(recv_msg, ?QOS_1) -> 'recv_msg.qos1'; +inc_qos_stats_key(recv_msg, ?QOS_2) -> 'recv_msg.qos2'; +%% for bad qos +inc_qos_stats_key(_, _) -> undefined. + +filter_disallowed_out(Packets) -> + lists:filter(fun is_datastream_out_pkt/1, Packets). + +is_datastream_out_pkt(#mqtt_packet{header = #mqtt_packet_header{type = Type}}) when + Type > 2 andalso Type < 12 +-> + true; +is_datastream_out_pkt(_) -> + false. +%% BUILD_WITHOUT_QUIC +-else. +-endif. diff --git a/apps/emqx/src/emqx_quic_stream.erl b/apps/emqx/src/emqx_quic_stream.erl index 567488862..f60345fe9 100644 --- a/apps/emqx/src/emqx_quic_stream.erl +++ b/apps/emqx/src/emqx_quic_stream.erl @@ -14,9 +14,18 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% MQTT/QUIC Stream +%% MQTT over QUIC +%% multistreams: This is the control stream. +%% single stream: This is the only main stream. +%% callbacks are from emqx_connection process rather than quicer_stream -module(emqx_quic_stream). +-ifndef(BUILD_WITHOUT_QUIC). + +-behaviour(quicer_remote_stream). + +-include("logger.hrl"). + %% emqx transport Callbacks -export([ type/1, @@ -31,44 +40,84 @@ sockname/1, peercert/1 ]). +-include_lib("quicer/include/quicer.hrl"). +-include_lib("emqx/include/emqx_quic.hrl"). -wait({ConnOwner, Conn}) -> +-type cb_ret() :: quicer_stream:cb_ret(). +-type cb_data() :: quicer_stream:cb_state(). +-type connection_handle() :: quicer:connection_handle(). +-type stream_handle() :: quicer:stream_handle(). + +-export([ + send_complete/3, + peer_send_shutdown/3, + peer_send_aborted/3, + peer_receive_aborted/3, + send_shutdown_complete/3, + stream_closed/3, + passive/3 +]). + +-export_type([socket/0]). + +-opaque socket() :: {quic, connection_handle(), stream_handle(), socket_info()}. + +-type socket_info() :: #{ + is_orphan => boolean(), + ctrl_stream_start_flags => quicer:stream_open_flags(), + %% and quicer:new_conn_props() + _ => _ +}. + +%%% For Accepting New Remote Stream +-spec wait({pid(), connection_handle(), socket_info()}) -> + {ok, socket()} | {error, enotconn}. +wait({ConnOwner, Conn, ConnInfo}) -> {ok, Conn} = quicer:async_accept_stream(Conn, []), ConnOwner ! {self(), stream_acceptor_ready}, receive - %% from msquic - {quic, new_stream, Stream} -> - {ok, {quic, Conn, Stream}}; + %% New incoming stream, this is a *control* stream + {quic, new_stream, Stream, #{is_orphan := IsOrphan, flags := StartFlags}} -> + SocketInfo = ConnInfo#{ + is_orphan => IsOrphan, + ctrl_stream_start_flags => StartFlags + }, + {ok, socket(Conn, Stream, SocketInfo)}; + %% connection closed event for stream acceptor + {quic, closed, undefined, undefined} -> + {error, enotconn}; + %% Connection owner process down {'EXIT', ConnOwner, _Reason} -> {error, enotconn} end. +-spec type(_) -> quic. type(_) -> quic. -peername({quic, Conn, _Stream}) -> +peername({quic, Conn, _Stream, _Info}) -> quicer:peername(Conn). -sockname({quic, Conn, _Stream}) -> +sockname({quic, Conn, _Stream, _Info}) -> quicer:sockname(Conn). peercert(_S) -> %% @todo but unsupported by msquic nossl. -getstat({quic, Conn, _Stream}, Stats) -> +getstat({quic, Conn, _Stream, _Info}, Stats) -> case quicer:getstat(Conn, Stats) of {error, _} -> {error, closed}; Res -> Res end. -setopts(Socket, Opts) -> +setopts({quic, _Conn, Stream, _Info}, Opts) -> lists:foreach( fun ({Opt, V}) when is_atom(Opt) -> - quicer:setopt(Socket, Opt, V); + quicer:setopt(Stream, Opt, V); (Opt) when is_atom(Opt) -> - quicer:setopt(Socket, Opt, true) + quicer:setopt(Stream, Opt, true) end, Opts ), @@ -84,9 +133,18 @@ getopts(_Socket, _Opts) -> {buffer, 80000} ]}. -fast_close({quic, _Conn, Stream}) -> - %% Flush send buffer, gracefully shutdown - quicer:async_shutdown_stream(Stream), +%% @TODO supply some App Error Code from caller +fast_close({ConnOwner, Conn, _ConnInfo}) when is_pid(ConnOwner) -> + %% handshake aborted. + _ = quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), + ok; +fast_close({quic, _Conn, Stream, _Info}) -> + %% Force flush + _ = quicer:async_shutdown_stream(Stream), + %% @FIXME Since we shutdown the control stream, we shutdown the connection as well + %% *BUT* Msquic does not flush the send buffer if we shutdown the connection after + %% gracefully shutdown the stream. + % quicer:async_shutdown_connection(Conn, ?QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0), ok. -spec ensure_ok_or_exit(atom(), list(term())) -> term(). @@ -102,8 +160,92 @@ ensure_ok_or_exit(Fun, Args = [Sock | _]) when is_atom(Fun), is_list(Args) -> Result end. -async_send({quic, _Conn, Stream}, Data, _Options) -> - case quicer:send(Stream, Data) of +async_send({quic, _Conn, Stream, _Info}, Data, _Options) -> + case quicer:async_send(Stream, Data, ?QUICER_SEND_FLAG_SYNC) of {ok, _Len} -> ok; + {error, X, Y} -> {error, {X, Y}}; Other -> Other end. + +%%% +%%% quicer stream callbacks +%%% + +-spec peer_receive_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). +peer_receive_aborted(Stream, ErrorCode, S) -> + _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + {ok, S}. + +-spec peer_send_aborted(stream_handle(), non_neg_integer(), cb_data()) -> cb_ret(). +peer_send_aborted(Stream, ErrorCode, S) -> + %% we abort receive with same reason + _ = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT, ErrorCode), + {ok, S}. + +-spec peer_send_shutdown(stream_handle(), undefined, cb_data()) -> cb_ret(). +peer_send_shutdown(Stream, undefined, S) -> + ok = quicer:async_shutdown_stream(Stream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 0), + {ok, S}. + +-spec send_complete(stream_handle(), boolean(), cb_data()) -> cb_ret(). +send_complete(_Stream, false, S) -> + {ok, S}; +send_complete(_Stream, true = _IsCancelled, S) -> + ?SLOG(error, #{message => "send cancelled"}), + {ok, S}. + +-spec send_shutdown_complete(stream_handle(), boolean(), cb_data()) -> cb_ret(). +send_shutdown_complete(_Stream, _IsGraceful, S) -> + {ok, S}. + +-spec passive(stream_handle(), undefined, cb_data()) -> cb_ret(). +passive(Stream, undefined, S) -> + case quicer:setopt(Stream, active, 10) of + ok -> ok; + Error -> ?SLOG(error, #{message => "set active error", error => Error}) + end, + {ok, S}. + +-spec stream_closed(stream_handle(), quicer:stream_closed_props(), cb_data()) -> + {{continue, term()}, cb_data()}. +stream_closed( + _Stream, + #{ + is_conn_shutdown := IsConnShutdown, + is_app_closing := IsAppClosing, + is_shutdown_by_app := IsAppShutdown, + is_closed_remotely := IsRemote, + status := Status, + error := Code + }, + S +) when + is_boolean(IsConnShutdown) andalso + is_boolean(IsAppClosing) andalso + is_boolean(IsAppShutdown) andalso + is_boolean(IsRemote) andalso + is_atom(Status) andalso + is_integer(Code) +-> + %% For now we fake a sock_closed for + %% emqx_connection:process_msg to append + %% a msg to be processed + Reason = + case Code of + ?MQTT_QUIC_CONN_NOERROR -> + normal; + _ -> + Status + end, + {{continue, {sock_closed, Reason}}, S}. + +%%% +%%% Internals +%%% +-spec socket(connection_handle(), stream_handle(), socket_info()) -> socket(). +socket(Conn, CtrlStream, Info) when is_map(Info) -> + {quic, Conn, CtrlStream, Info}. + +%% BUILD_WITHOUT_QUIC +-else. +-endif. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index ba813d2c8..8d24e6937 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -268,7 +268,7 @@ fields("persistent_session_store") -> sc( duration(), #{ - default => "1h", + default => <<"1h">>, desc => ?DESC(persistent_session_store_max_retain_undelivered) } )}, @@ -276,7 +276,7 @@ fields("persistent_session_store") -> sc( duration(), #{ - default => "1h", + default => <<"1h">>, desc => ?DESC(persistent_session_store_message_gc_interval) } )}, @@ -284,7 +284,7 @@ fields("persistent_session_store") -> sc( duration(), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(persistent_session_store_session_message_gc_interval) } )} @@ -352,7 +352,7 @@ fields("authz_cache") -> sc( duration(), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(fields_cache_ttl) } )} @@ -363,7 +363,7 @@ fields("mqtt") -> sc( hoconsc:union([infinity, duration()]), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC(mqtt_idle_timeout) } )}, @@ -371,7 +371,7 @@ fields("mqtt") -> sc( bytesize(), #{ - default => "1MB", + default => <<"1MB">>, desc => ?DESC(mqtt_max_packet_size) } )}, @@ -507,7 +507,7 @@ fields("mqtt") -> sc( duration(), #{ - default => "30s", + default => <<"30s">>, desc => ?DESC(mqtt_retry_interval) } )}, @@ -523,7 +523,7 @@ fields("mqtt") -> sc( duration(), #{ - default => "300s", + default => <<"300s">>, desc => ?DESC(mqtt_await_rel_timeout) } )}, @@ -531,7 +531,7 @@ fields("mqtt") -> sc( duration(), #{ - default => "2h", + default => <<"2h">>, desc => ?DESC(mqtt_session_expiry_interval) } )}, @@ -617,7 +617,7 @@ fields("flapping_detect") -> sc( duration(), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(flapping_detect_window_time) } )}, @@ -625,7 +625,7 @@ fields("flapping_detect") -> sc( duration(), #{ - default => "5m", + default => <<"5m">>, desc => ?DESC(flapping_detect_ban_time) } )} @@ -652,7 +652,7 @@ fields("force_shutdown") -> sc( wordsize(), #{ - default => "32MB", + default => <<"32MB">>, desc => ?DESC(force_shutdown_max_heap_size), validator => fun ?MODULE:validate_heap_size/1 } @@ -715,7 +715,7 @@ fields("conn_congestion") -> sc( duration(), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(conn_congestion_min_alarm_sustain_duration) } )} @@ -739,7 +739,7 @@ fields("force_gc") -> sc( bytesize(), #{ - default => "16MB", + default => <<"16MB">>, desc => ?DESC(force_gc_bytes) } )} @@ -845,16 +845,21 @@ fields("mqtt_wss_listener") -> ]; fields("mqtt_quic_listener") -> [ - %% TODO: ensure cacertfile is configurable {"certfile", sc( string(), - #{desc => ?DESC(fields_mqtt_quic_listener_certfile)} + #{ + %% TODO: deprecated => {since, "5.1.0"} + desc => ?DESC(fields_mqtt_quic_listener_certfile) + } )}, {"keyfile", sc( string(), - #{desc => ?DESC(fields_mqtt_quic_listener_keyfile)} + %% TODO: deprecated => {since, "5.1.0"} + #{ + desc => ?DESC(fields_mqtt_quic_listener_keyfile) + } )}, {"ciphers", ciphers_schema(quic)}, {"idle_timeout", @@ -869,7 +874,7 @@ fields("mqtt_quic_listener") -> sc( duration_ms(), #{ - default => "10s", + default => <<"10s">>, desc => ?DESC(fields_mqtt_quic_listener_handshake_idle_timeout) } )}, @@ -880,6 +885,14 @@ fields("mqtt_quic_listener") -> default => 0, desc => ?DESC(fields_mqtt_quic_listener_keep_alive_interval) } + )}, + {"ssl_options", + sc( + ref("listener_quic_ssl_opts"), + #{ + required => false, + desc => ?DESC(fields_mqtt_quic_listener_ssl_options) + } )} ] ++ base_listener(14567); fields("ws_opts") -> @@ -888,7 +901,7 @@ fields("ws_opts") -> sc( string(), #{ - default => "/mqtt", + default => <<"/mqtt">>, desc => ?DESC(fields_ws_opts_mqtt_path) } )}, @@ -912,7 +925,7 @@ fields("ws_opts") -> sc( duration(), #{ - default => "7200s", + default => <<"7200s">>, desc => ?DESC(fields_ws_opts_idle_timeout) } )}, @@ -936,7 +949,7 @@ fields("ws_opts") -> sc( comma_separated_list(), #{ - default => "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5", + default => <<"mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5">>, desc => ?DESC(fields_ws_opts_supported_subprotocols) } )}, @@ -968,7 +981,7 @@ fields("ws_opts") -> sc( string(), #{ - default => "x-forwarded-for", + default => <<"x-forwarded-for">>, desc => ?DESC(fields_ws_opts_proxy_address_header) } )}, @@ -976,7 +989,7 @@ fields("ws_opts") -> sc( string(), #{ - default => "x-forwarded-port", + default => <<"x-forwarded-port">>, desc => ?DESC(fields_ws_opts_proxy_port_header) } )}, @@ -1008,7 +1021,7 @@ fields("tcp_opts") -> sc( duration(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC(fields_tcp_opts_send_timeout) } )}, @@ -1049,7 +1062,7 @@ fields("tcp_opts") -> sc( bytesize(), #{ - default => "1MB", + default => <<"1MB">>, desc => ?DESC(fields_tcp_opts_high_watermark) } )}, @@ -1090,6 +1103,8 @@ fields("listener_wss_opts") -> }, true ); +fields("listener_quic_ssl_opts") -> + server_ssl_opts_schema(#{}, false); fields("ssl_client_opts") -> client_ssl_opts_schema(#{}); fields("deflate_opts") -> @@ -1260,7 +1275,7 @@ fields("sys_topics") -> sc( hoconsc:union([disabled, duration()]), #{ - default => "1m", + default => <<"1m">>, desc => ?DESC(sys_msg_interval) } )}, @@ -1268,7 +1283,7 @@ fields("sys_topics") -> sc( hoconsc:union([disabled, duration()]), #{ - default => "30s", + default => <<"30s">>, desc => ?DESC(sys_heartbeat_interval) } )}, @@ -1337,7 +1352,7 @@ fields("sysmon_vm") -> sc( duration(), #{ - default => "30s", + default => <<"30s">>, desc => ?DESC(sysmon_vm_process_check_interval) } )}, @@ -1345,7 +1360,7 @@ fields("sysmon_vm") -> sc( percent(), #{ - default => "80%", + default => <<"80%">>, desc => ?DESC(sysmon_vm_process_high_watermark) } )}, @@ -1353,7 +1368,7 @@ fields("sysmon_vm") -> sc( percent(), #{ - default => "60%", + default => <<"60%">>, desc => ?DESC(sysmon_vm_process_low_watermark) } )}, @@ -1369,7 +1384,7 @@ fields("sysmon_vm") -> sc( hoconsc:union([disabled, duration()]), #{ - default => "240ms", + default => <<"240ms">>, desc => ?DESC(sysmon_vm_long_schedule) } )}, @@ -1377,7 +1392,7 @@ fields("sysmon_vm") -> sc( hoconsc:union([disabled, bytesize()]), #{ - default => "32MB", + default => <<"32MB">>, desc => ?DESC(sysmon_vm_large_heap) } )}, @@ -1404,7 +1419,7 @@ fields("sysmon_os") -> sc( duration(), #{ - default => "60s", + default => <<"60s">>, desc => ?DESC(sysmon_os_cpu_check_interval) } )}, @@ -1412,7 +1427,7 @@ fields("sysmon_os") -> sc( percent(), #{ - default => "80%", + default => <<"80%">>, desc => ?DESC(sysmon_os_cpu_high_watermark) } )}, @@ -1420,7 +1435,7 @@ fields("sysmon_os") -> sc( percent(), #{ - default => "60%", + default => <<"60%">>, desc => ?DESC(sysmon_os_cpu_low_watermark) } )}, @@ -1428,7 +1443,7 @@ fields("sysmon_os") -> sc( hoconsc:union([disabled, duration()]), #{ - default => "60s", + default => <<"60s">>, desc => ?DESC(sysmon_os_mem_check_interval) } )}, @@ -1436,7 +1451,7 @@ fields("sysmon_os") -> sc( percent(), #{ - default => "70%", + default => <<"70%">>, desc => ?DESC(sysmon_os_sysmem_high_watermark) } )}, @@ -1444,7 +1459,7 @@ fields("sysmon_os") -> sc( percent(), #{ - default => "5%", + default => <<"5%">>, desc => ?DESC(sysmon_os_procmem_high_watermark) } )} @@ -1465,7 +1480,7 @@ fields("sysmon_top") -> emqx_schema:duration(), #{ mapping => "system_monitor.top_sample_interval", - default => "2s", + default => <<"2s">>, desc => ?DESC(sysmon_top_sample_interval) } )}, @@ -1484,7 +1499,7 @@ fields("sysmon_top") -> #{ mapping => "system_monitor.db_hostname", desc => ?DESC(sysmon_top_db_hostname), - default => "" + default => <<>> } )}, {"db_port", @@ -1501,7 +1516,7 @@ fields("sysmon_top") -> string(), #{ mapping => "system_monitor.db_username", - default => "system_monitor", + default => <<"system_monitor">>, desc => ?DESC(sysmon_top_db_username) } )}, @@ -1510,7 +1525,7 @@ fields("sysmon_top") -> binary(), #{ mapping => "system_monitor.db_password", - default => "system_monitor_password", + default => <<"system_monitor_password">>, desc => ?DESC(sysmon_top_db_password), converter => fun password_converter/2, sensitive => true @@ -1521,7 +1536,7 @@ fields("sysmon_top") -> string(), #{ mapping => "system_monitor.db_name", - default => "postgres", + default => <<"postgres">>, desc => ?DESC(sysmon_top_db_name) } )} @@ -1551,7 +1566,7 @@ fields("alarm") -> sc( duration(), #{ - default => "24h", + default => <<"24h">>, example => "24h", desc => ?DESC(alarm_validity_period) } @@ -1590,7 +1605,7 @@ mqtt_listener(Bind) -> duration(), #{ desc => ?DESC(mqtt_listener_proxy_protocol_timeout), - default => "3s" + default => <<"3s">> } )}, {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(listener)} @@ -1769,6 +1784,12 @@ desc("listener_ssl_opts") -> "Socket options for SSL connections."; desc("listener_wss_opts") -> "Socket options for WebSocket/SSL connections."; +desc("fields_mqtt_quic_listener_certfile") -> + "Path to the certificate file. Will be deprecated in 5.1, use '.ssl_options.certfile' instead."; +desc("fields_mqtt_quic_listener_keyfile") -> + "Path to the secret key file. Will be deprecated in 5.1, use '.ssl_options.keyfile' instead."; +desc("listener_quic_ssl_opts") -> + "TLS options for QUIC transport."; desc("ssl_client_opts") -> "Socket options for SSL clients."; desc("deflate_opts") -> @@ -1929,6 +1950,15 @@ common_ssl_opts_schema(Defaults) -> default => Df("secure_renegotiate", true), desc => ?DESC(common_ssl_opts_schema_secure_renegotiate) } + )}, + + {"hibernate_after", + sc( + duration(), + #{ + default => Df("hibernate_after", <<"5s">>), + desc => ?DESC(common_ssl_opts_schema_hibernate_after) + } )} ]. @@ -1976,7 +2006,7 @@ server_ssl_opts_schema(Defaults, IsRanchListener) -> sc( duration(), #{ - default => Df("handshake_timeout", "15s"), + default => Df("handshake_timeout", <<"15s">>), desc => ?DESC(server_ssl_opts_schema_handshake_timeout) } )} diff --git a/apps/emqx/src/emqx_vm.erl b/apps/emqx/src/emqx_vm.erl index f80d18a3a..0d861f671 100644 --- a/apps/emqx/src/emqx_vm.erl +++ b/apps/emqx/src/emqx_vm.erl @@ -24,7 +24,6 @@ get_system_info/1, get_memory/0, get_memory/2, - mem_info/0, loads/0 ]). @@ -226,12 +225,6 @@ convert_allocated_areas({Key, Value1, Value2}) -> convert_allocated_areas({Key, Value}) -> {Key, Value}. -mem_info() -> - Dataset = memsup:get_system_memory_data(), - Total = proplists:get_value(total_memory, Dataset), - Free = proplists:get_value(free_memory, Dataset), - [{total_memory, Total}, {used_memory, Total - Free}]. - %%%% erlang vm scheduler_usage fun copied from recon scheduler_usage(Interval) when is_integer(Interval) -> %% We start and stop the scheduler_wall_time system flag diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 954151efa..5149b8b8a 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -22,6 +22,8 @@ -export([ all/1, + init_per_testcase/3, + end_per_testcase/3, boot_modules/1, start_apps/1, start_apps/2, @@ -150,6 +152,19 @@ all(Suite) -> string:substr(atom_to_list(F), 1, 2) == "t_" ]). +init_per_testcase(Module, TestCase, Config) -> + case erlang:function_exported(Module, TestCase, 2) of + true -> Module:TestCase(init, Config); + false -> Config + end. + +end_per_testcase(Module, TestCase, Config) -> + case erlang:function_exported(Module, TestCase, 2) of + true -> Module:TestCase('end', Config); + false -> ok + end, + Config. + %% set emqx app boot modules -spec boot_modules(all | list(atom())) -> ok. boot_modules(Mods) -> @@ -499,8 +514,8 @@ ensure_quic_listener(Name, UdpPort) -> application:ensure_all_started(quicer), Conf = #{ acceptors => 16, - bind => {{0, 0, 0, 0}, UdpPort}, - certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"), + bind => UdpPort, + ciphers => [ "TLS_AES_256_GCM_SHA384", @@ -509,7 +524,10 @@ ensure_quic_listener(Name, UdpPort) -> ], enabled => true, idle_timeout => 15000, - keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem"), + ssl_options => #{ + certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"), + keyfile => filename:join(code:lib_dir(emqx), "etc/certs/key.pem") + }, limiter => #{}, max_connections => 1024000, mountpoint => <<>>, diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index 6a7cd2791..015439587 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -138,6 +138,41 @@ t_restart_listeners(_) -> ok = emqx_listeners:restart(), ok = emqx_listeners:stop(). +t_restart_listeners_with_hibernate_after_disabled(_Config) -> + OldLConf = emqx_config:get([listeners]), + maps:foreach( + fun(LType, Listeners) -> + maps:foreach( + fun(Name, Opts) -> + case maps:is_key(ssl_options, Opts) of + true -> + emqx_config:put( + [ + listeners, + LType, + Name, + ssl_options, + hibernate_after + ], + undefined + ); + _ -> + skip + end + end, + Listeners + ) + end, + OldLConf + ), + ok = emqx_listeners:start(), + ok = emqx_listeners:stop(), + %% flakyness: eaddrinuse + timer:sleep(timer:seconds(2)), + ok = emqx_listeners:restart(), + ok = emqx_listeners:stop(), + emqx_config:put([listeners], OldLConf). + t_max_conns_tcp(_) -> %% Note: Using a string representation for the bind address like %% "127.0.0.1" does not work diff --git a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl index 7e97c5bf4..d3de74f72 100644 --- a/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_protocol_v5_SUITE.erl @@ -905,7 +905,7 @@ t_shared_subscriptions_client_terminates_when_qos_eq_2(Config) -> emqtt, connected, fun - (cast, ?PUBLISH_PACKET(?QOS_2, _PacketId), _State) -> + (cast, {?PUBLISH_PACKET(?QOS_2, _PacketId), _Via}, _State) -> ok = counters:add(CRef, 1, 1), {stop, {shutdown, for_testing}}; (Arg1, ARg2, Arg3) -> diff --git a/apps/emqx/test/emqx_quic_multistreams_SUITE.erl b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl new file mode 100644 index 000000000..17ba85da7 --- /dev/null +++ b/apps/emqx/test/emqx_quic_multistreams_SUITE.erl @@ -0,0 +1,1986 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_quic_multistreams_SUITE). + +-ifndef(BUILD_WITHOUT_QUIC). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("quicer/include/quicer.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). + +suite() -> + [{timetrap, {seconds, 30}}]. + +all() -> + [ + {group, mstream}, + {group, shutdown}, + {group, misc} + ]. + +groups() -> + [ + {mstream, [], [{group, profiles}]}, + + {profiles, [], [ + {group, profile_low_latency}, + {group, profile_max_throughput} + ]}, + {profile_low_latency, [], [ + {group, pub_qos0}, + {group, pub_qos1}, + {group, pub_qos2} + ]}, + {profile_max_throughput, [], [ + {group, pub_qos0}, + {group, pub_qos1}, + {group, pub_qos2} + ]}, + {pub_qos0, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {pub_qos1, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {pub_qos2, [], [ + {group, sub_qos0}, + {group, sub_qos1}, + {group, sub_qos2} + ]}, + {sub_qos0, [{group, qos}]}, + {sub_qos1, [{group, qos}]}, + {sub_qos2, [{group, qos}]}, + {qos, [ + t_multi_streams_sub, + t_multi_streams_pub_5x100, + t_multi_streams_pub_parallel, + t_multi_streams_pub_parallel_no_blocking, + t_multi_streams_sub_pub_async, + t_multi_streams_sub_pub_sync, + t_multi_streams_unsub, + t_multi_streams_corr_topic, + t_multi_streams_unsub_via_other, + t_multi_streams_dup_sub, + t_multi_streams_packet_boundary, + t_multi_streams_packet_malform, + t_multi_streams_kill_sub_stream, + t_multi_streams_packet_too_large, + t_multi_streams_sub_0_rtt, + t_multi_streams_sub_0_rtt_large_payload, + t_multi_streams_sub_0_rtt_stream_data_cont, + t_conn_change_client_addr + ]}, + + {shutdown, [ + {group, graceful_shutdown}, + {group, abort_recv_shutdown}, + {group, abort_send_shutdown}, + {group, abort_send_recv_shutdown} + ]}, + + {graceful_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_recv_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_send_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + {abort_send_recv_shutdown, [ + {group, ctrl_stream_shutdown}, + {group, data_stream_shutdown} + ]}, + + {ctrl_stream_shutdown, [ + t_multi_streams_shutdown_ctrl_stream, + t_multi_streams_shutdown_ctrl_stream_then_reconnect, + t_multi_streams_remote_shutdown, + t_multi_streams_emqx_ctrl_kill, + t_multi_streams_emqx_ctrl_exit_normal, + t_multi_streams_remote_shutdown_with_reconnect + ]}, + + {data_stream_shutdown, [ + t_multi_streams_shutdown_pub_data_stream, + t_multi_streams_shutdown_sub_data_stream + ]}, + {misc, [ + t_conn_silent_close, + t_client_conn_bump_streams, + t_olp_true, + t_olp_reject, + t_conn_resume, + t_conn_without_ctrl_stream + ]} + ]. + +init_per_suite(Config) -> + emqx_common_test_helpers:start_apps([]), + UdpPort = 14567, + start_emqx_quic(UdpPort), + %% Turn off force_shutdown policy. + ShutdownPolicy = emqx_config:get_zone_conf(default, [force_shutdown]), + ct:pal("force shutdown config: ~p", [ShutdownPolicy]), + emqx_config:put_zone_conf(default, [force_shutdown], ShutdownPolicy#{enable := false}), + [{shutdown_policy, ShutdownPolicy}, {port, UdpPort}, {pub_qos, 0}, {sub_qos, 0} | Config]. + +end_per_suite(Config) -> + emqx_config:put_zone_conf(default, [force_shutdown], ?config(shutdown_policy, Config)), + ok. + +init_per_group(pub_qos0, Config) -> + [{pub_qos, 0} | Config]; +init_per_group(sub_qos0, Config) -> + [{sub_qos, 0} | Config]; +init_per_group(pub_qos1, Config) -> + [{pub_qos, 1} | Config]; +init_per_group(sub_qos1, Config) -> + [{sub_qos, 1} | Config]; +init_per_group(pub_qos2, Config) -> + [{pub_qos, 2} | Config]; +init_per_group(sub_qos2, Config) -> + [{sub_qos, 2} | Config]; +init_per_group(abort_send_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND} | Config]; +init_per_group(abort_recv_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE} | Config]; +init_per_group(abort_send_recv_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT} | Config]; +init_per_group(graceful_shutdown, Config) -> + [{stream_shutdown_flag, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL} | Config]; +init_per_group(profile_max_throughput, Config) -> + quicer:reg_open(quic_execution_profile_type_max_throughput), + Config; +init_per_group(profile_low_latency, Config) -> + quicer:reg_open(quic_execution_profile_low_latency), + Config; +init_per_group(_, Config) -> + Config. + +end_per_group(_, Config) -> + Config. + +init_per_testcase(_, Config) -> + emqx_common_test_helpers:start_apps([]), + Config. + +t_quic_sock(Config) -> + Port = 4567, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, Sock} = emqtt_quic:connect( + "localhost", + Port, + [{alpn, ["mqtt"]}, {active, false}], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + quic_server:stop(Server). + +t_quic_sock_fail(_Config) -> + Port = 4567, + Error1 = + {error, + {transport_down, #{ + error => 2, + status => connection_refused + }}}, + Error2 = {error, {transport_down, #{error => 1, status => unreachable}}}, + case + emqtt_quic:connect( + "localhost", + Port, + [{alpn, ["mqtt"]}, {active, false}], + 3000 + ) + of + Error1 -> + ok; + Error2 -> + ok; + Other -> + ct:fail("unexpected return ~p", [Other]) + end. + +t_0_rtt(Config) -> + Port = 4568, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {quic_event_mask, 1} + ], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + NST = + receive + {quic, nst_received, Conn, Ticket} -> + Ticket + end, + {ok, Sock2} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {nst, NST} + ], + 3000 + ), + send_and_recv_with(Sock2), + ok = emqtt_quic:close(Sock2), + quic_server:stop(Server). + +t_0_rtt_fail(Config) -> + Port = 4569, + SslOpts = [ + {cert, certfile(Config)}, + {key, keyfile(Config)}, + {idle_timeout_ms, 10000}, + % QUIC_SERVER_RESUME_AND_ZERORTT + {server_resumption_level, 2}, + {peer_bidi_stream_count, 10}, + {alpn, ["mqtt"]} + ], + Server = quic_server:start_link(Port, SslOpts), + timer:sleep(500), + {ok, {quic, Conn, _Stream} = Sock} = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {quic_event_mask, 1} + ], + 3000 + ), + send_and_recv_with(Sock), + ok = emqtt_quic:close(Sock), + <<_Head:16, Left/binary>> = + receive + {quic, nst_received, Conn, Ticket} when is_binary(Ticket) -> + Ticket + end, + + Error = {error, {not_found, invalid_parameter}}, + Error = emqtt_quic:connect( + "localhost", + Port, + [ + {alpn, ["mqtt"]}, + {active, false}, + {nst, Left} + ], + 3000 + ), + quic_server:stop(Server). + +t_multi_streams_sub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + case emqtt:publish(C, Topic, <<"qos 2 1">>, PubQos) of + ok when PubQos == 0 -> ok; + {ok, _} -> ok + end, + receive + {publish, #{ + client_pid := C, + payload := <<"qos 2 1">>, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C). + +t_multi_streams_pub_5x100(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + PubVias = lists:map( + fun(_N) -> + {ok, Via} = emqtt:start_data_stream(C, []), + Via + end, + lists:seq(1, 5) + ), + CtrlVia = proplists:get_value(socket, emqtt:info(C)), + [ + begin + case emqtt:publish_via(C, PVia, Topic, #{}, <<"stream data ", N>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, _} -> ok + end, + 0 == (N rem 10) andalso timer:sleep(10) + end + || %% also publish on control stream + N <- lists:seq(1, 100), + PVia <- [CtrlVia | PubVias] + ], + ?assert(timeout =/= recv_pub(600)), + ok = emqtt:disconnect(C). + +t_multi_streams_pub_parallel(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data", _/binary>>, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + Payloads = [P || {publish, #{payload := P}} <- PubRecvs], + ?assert( + [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse + [<<"stream data 2">>, <<"stream data 1">>] == Payloads + ), + ok = emqtt:disconnect(C). + +%% @doc test two pub streams, one send incomplete MQTT packet() can not block another. +t_multi_streams_pub_parallel_no_blocking(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId2 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + Drop = <<"stream data 1">>, + meck:new(emqtt_quic, [passthrough, no_history]), + meck:expect(emqtt_quic, send, fun(Sock, IoList) -> + case lists:last(IoList) == Drop of + true -> + ct:pal("meck droping ~p", [Drop]), + meck:passthrough([Sock, IoList -- [Drop]]); + false -> + meck:passthrough([Sock, IoList]) + end + end), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + Drop, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + meck:unload(emqtt_quic), + ?assertEqual(timeout, recv_pub(1)), + ok = emqtt:disconnect(C). + +t_multi_streams_packet_boundary(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + PktId3 = calc_pkt_id(RecQos, 3), + Topic = atom_to_binary(?FUNCTION_NAME), + + %% make quicer to batch job + quicer:reg_open(quic_execution_profile_type_max_throughput), + + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + ThisFunB = atom_to_binary(?FUNCTION_NAME), + LargePart3 = iolist_to_binary([ + <> + || N <- lists:seq(1, 20000) + ]), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + LargePart3, + [{qos, PubQos}], + undefined + ), + timer:sleep(300), + PubRecvs = recv_pub(3, [], 1000), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 1">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId3, + payload := _LargePart3_TO_BE_CHECKED, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + {publish, #{payload := LargePart3Recv}} = lists:last(PubRecvs), + CommonLen = binary:longest_common_prefix([LargePart3Recv, LargePart3]), + Size3 = byte_size(LargePart3), + case Size3 - CommonLen of + 0 -> + ok; + Left -> + ct:fail( + "unmatched large payload: offset: ~p ~n send: ~p ~n recv ~p", + [ + CommonLen, + binary:part(LargePart3, {CommonLen, Left}), + binary:part(LargePart3Recv, {CommonLen, Left}) + ] + ) + end, + ok = emqtt:disconnect(C). + +%% @doc test that one malformed stream will not close the entire connection +t_multi_streams_packet_malform(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + PktId3 = calc_pkt_id(RecQos, 3), + Topic = atom_to_binary(?FUNCTION_NAME), + + %% make quicer to batch job + quicer:reg_open(quic_execution_profile_type_max_throughput), + + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + + {ok, {quic, _Conn, MalformStream}} = emqtt:start_data_stream(C, []), + {ok, _} = quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>), + + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + LargePart3 = binary:copy(atom_to_binary(?FUNCTION_NAME), 2000), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + LargePart3, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(3), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 1">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId3, + payload := LargePart3, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + + case quicer:send(MalformStream, <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0>>) of + {ok, 10} -> ok; + {error, cancelled} -> ok; + {error, stm_send_error, aborted} -> ok + end, + + timer:sleep(200), + ?assert(is_list(emqtt:info(C))), + + {error, stm_send_error, aborted} = quicer:send(MalformStream, <<1, 2, 3, 4, 5, 6, 7, 8, 9, 0>>), + + timer:sleep(200), + ?assert(is_list(emqtt:info(C))), + + ok = emqtt:disconnect(C). + +t_multi_streams_packet_too_large(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + Topic = atom_to_binary(?FUNCTION_NAME), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + PktId3 = calc_pkt_id(RecQos, 3), + + OldMax = emqx_config:get_zone_conf(default, [mqtt, max_packet_size]), + emqx_config:put_zone_conf(default, [mqtt, max_packet_size], 1000), + + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 1">>, + qos := RecQos, + topic := Topic + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<"stream data 2">>, + qos := RecQos, + topic := Topic + }} + ], + PubRecvs + ), + + {ok, PubVia2} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia2, + Topic, + binary:copy(<<"too large">>, 200), + [{qos, PubQos}], + undefined + ), + timer:sleep(200), + ?assert(is_list(emqtt:info(C))), + + timeout = recv_pub(1), + + %% send large payload on stream 1 + ok = emqtt:publish_async( + C, + PubVia, + Topic, + binary:copy(<<"too large">>, 200), + [{qos, PubQos}], + undefined + ), + timer:sleep(200), + timeout = recv_pub(1), + ?assert(is_list(emqtt:info(C))), + + %% Connection could be kept + {error, stm_send_error, _} = quicer:send(via_stream(PubVia), <<1>>), + {error, stm_send_error, _} = quicer:send(via_stream(PubVia2), <<1>>), + %% We could send data over new stream + {ok, PubVia3} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia3, + Topic, + <<"stream data 3">>, + [{qos, PubQos}], + undefined + ), + [ + {publish, #{ + client_pid := C, + packet_id := PktId3, + payload := <<"stream data 3">>, + qos := RecQos, + topic := Topic + }} + ] = recv_pub(1), + timer:sleep(200), + + ?assert(is_list(emqtt:info(C))), + + emqx_config:put_zone_conf(default, [mqtt, max_packet_size], OldMax), + ok = emqtt:disconnect(C). + +t_conn_change_client_addr(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe(C, #{}, [{Topic, [{qos, SubQos}]}]), + + {ok, {quic, Conn, _} = PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := _PktId1, + payload := <<"stream data 1">>, + qos := RecQos + }} + ], + recv_pub(1) + ), + NewPort = select_port(), + {ok, OldAddr} = quicer:sockname(Conn), + ?assertEqual( + ok, quicer:setopt(Conn, param_conn_local_address, "127.0.0.1:" ++ integer_to_list(NewPort)) + ), + {ok, NewAddr} = quicer:sockname(Conn), + ct:pal("NewAddr: ~p, Old Addr: ~p", [NewAddr, OldAddr]), + ?assertNotEqual(OldAddr, NewAddr), + ?assert(is_list(emqtt:info(C))), + ok = emqtt:disconnect(C). + +t_multi_streams_sub_pub_async(Config) -> + Topic = atom_to_binary(?FUNCTION_NAME), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, _, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + <<"stream data 1">>, + [{qos, PubQos}], + undefined + ), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic2, + <<"stream data 2">>, + [{qos, PubQos}], + undefined + ), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data", _/binary>>, + qos := RecQos + }} + ], + PubRecvs + ), + Payloads = [P || {publish, #{payload := P}} <- PubRecvs], + ?assert( + [<<"stream data 1">>, <<"stream data 2">>] == Payloads orelse + [<<"stream data 2">>, <<"stream data 1">>] == Payloads + ), + ok = emqtt:disconnect(C). + +t_multi_streams_sub_pub_sync(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + Via1 = undefined, + ok; + {ok, #{reason_code := 0, via := Via1}} -> + ok + end, + case + emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<"stream data 4">>, [ + {qos, PubQos} + ]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := Via2}} -> + ?assert(Via1 =/= Via2), + ok + end, + ct:pal("SVia1: ~p, SVia2: ~p", [SVia1, SVia2]), + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos, + via := SVia1 + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 4">>, + qos := RecQos, + via := SVia2 + }} + ], + lists:sort(PubRecvs) + ), + ok = emqtt:disconnect(C). + +t_multi_streams_dup_sub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia1}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + #{data_stream_socks := [{quic, _Conn, SubStream} | _]} = proplists:get_value( + extra, emqtt:info(C) + ), + ?assertEqual(2, length(emqx_broker:subscribers(Topic))), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<"stream data 3">>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _Via1}} -> + ok + end, + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<"stream data 3">>, + qos := RecQos + }} + ], + lists:sort(PubRecvs) + ), + + RecvVias = [Via || {publish, #{via := Via}} <- PubRecvs], + + ct:pal("~p, ~p, ~n recv from: ~p~n", [SVia1, SVia2, PubRecvs]), + %% Can recv in any order + ?assert([SVia1, SVia2] == RecvVias orelse [SVia2, SVia1] == RecvVias), + + %% Shutdown one stream + quicer:async_shutdown_stream(SubStream, ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 500), + timer:sleep(100), + + ?assertEqual(1, length(emqx_broker:subscribers(Topic))), + + ok = emqtt:disconnect(C). + +t_multi_streams_corr_topic(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _Via}} -> + ok + end, + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + ?assert(PubVia =/= SubVia), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := PubVia}} -> ok + end, + PubRecvs = recv_pub(2), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }}, + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<6, 7, 8, 9>>, + qos := RecQos + }} + ], + PubRecvs + ), + ok = emqtt:disconnect(C). + +t_multi_streams_unsub(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SubVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _PVia}} -> + ok + end, + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + ?assert(PubVia =/= SubVia), + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + emqtt:unsubscribe_via(C, SubVia, Topic), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 16, via := PubVia, reason_code_name := no_matching_subscribers}} -> + ok + end, + + timeout = recv_pub(1), + ok = emqtt:disconnect(C). + +t_multi_streams_kill_sub_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + [TopicStreamOwner] = emqx_broker:subscribers(Topic), + exit(TopicStreamOwner, kill), + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := Code, via := _PVia}} when Code == 0 orelse Code == 16 -> + ok + end, + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic2, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> + ok; + {ok, #{reason_code := 0, via := _PVia2}} -> + ok + end, + + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + topic := Topic2, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + recv_pub(1) + ), + ?assertEqual(timeout, recv_pub(1)), + ok. + +t_multi_streams_unsub_via_other(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + PktId2 = calc_pkt_id(RecQos, 2), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + + %% Unsub topic1 via stream2 should fail with error code 17: "No subscription existed" + {ok, #{via := SVia2}, [17]} = emqtt:unsubscribe_via(C, SVia2, Topic), + + case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia2}} -> ok + end, + + PubRecvs2 = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId2, + payload := <<6, 7, 8, 9>>, + qos := RecQos + }} + ], + PubRecvs2 + ), + ok = emqtt:disconnect(C). + +t_multi_streams_shutdown_pub_data_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia =/= SVia2), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + #{data_stream_socks := [PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + {quic, _Conn, DataStream} = PubVia, + quicer:shutdown_stream(DataStream, ?config(stream_shutdown_flag, Config), 500, 100), + timer:sleep(500), + %% Still alive + ?assert(is_list(emqtt:info(C))), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ). + +t_multi_streams_shutdown_sub_data_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia =/= SVia2), + {quic, _Conn, DataStream} = SVia2, + quicer:shutdown_stream(DataStream, ?QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, 500, 100), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + #{data_stream_socks := [_PubVia | _]} = proplists:get_value(extra, emqtt:info(C)), + timer:sleep(500), + %% Still alive + ?assert(is_list(emqtt:info(C))). + +t_multi_streams_shutdown_ctrl_stream(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + unlink(C), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := _SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := _SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + Flag = ?config(stream_shutdown_flag, Config), + AppErrorCode = + case Flag of + ?QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL -> 0; + _ -> 500 + end, + quicer:shutdown_stream(Ctrlstream, Flag, AppErrorCode, 1000), + timer:sleep(500), + %% Client should be closed + ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_shutdown_ctrl_stream_then_reconnect(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, true}, + {clean_start, false}, + {clientid, atom_to_binary(?FUNCTION_NAME)}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + quicer:shutdown_stream(Ctrlstream, ?config(stream_shutdown_flag, Config), 500, 100), + timer:sleep(200), + %% Client should be closed + ?assert(is_list(emqtt:info(C))). + +t_multi_streams_emqx_ctrl_kill(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, false}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + ClientId = proplists:get_value(clientid, emqtt:info(C)), + [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId), + exit(TransPid, kill), + + timer:sleep(200), + %% Client should be closed + ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_emqx_ctrl_exit_normal(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, false}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + ClientId = proplists:get_value(clientid, emqtt:info(C)), + [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId), + + emqx_connection:stop(TransPid), + timer:sleep(200), + %% Client exit normal. + ?assertMatch({'EXIT', {normal, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_remote_shutdown(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, false}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + + ok = stop_emqx(), + start_emqx_quic(?config(port, Config)), + timer:sleep(200), + %% Client should be closed + ?assertMatch({'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}}, catch emqtt:info(C)). + +t_multi_streams_remote_shutdown_with_reconnect(Config) -> + erlang:process_flag(trap_exit, true), + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + PktId1 = calc_pkt_id(RecQos, 1), + + Topic = atom_to_binary(?FUNCTION_NAME), + Topic2 = <>, + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {reconnect, true}, + {clean_start, false}, + {clientid, atom_to_binary(?FUNCTION_NAME)}, + %% speedup test + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {ok, #{via := SVia}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, #{via := SVia2}, [SubQos]} = emqtt:subscribe_via(C, {new_data_stream, []}, #{}, [ + {Topic2, [{qos, SubQos}]} + ]), + + ?assert(SVia2 =/= SVia), + + case + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, PubQos}]) + of + ok when PubQos == 0 -> ok; + {ok, #{reason_code := 0, via := _PVia}} -> ok + end, + + PubRecvs = recv_pub(1), + ?assertMatch( + [ + {publish, #{ + client_pid := C, + packet_id := PktId1, + payload := <<1, 2, 3, 4, 5>>, + qos := RecQos + }} + ], + PubRecvs + ), + + {quic, _Conn, _Ctrlstream} = proplists:get_value(socket, emqtt:info(C)), + + ok = stop_emqx(), + + timer:sleep(200), + + start_emqx_quic(?config(port, Config)), + %% Client should be closed + ?assert(is_list(emqtt:info(C))). + +t_conn_silent_close(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + %% quic idle timeout + 1s + timer:sleep(16000), + Topic = atom_to_binary(?FUNCTION_NAME), + ?assertException( + exit, + noproc, + emqtt:publish_via(C, {new_data_stream, []}, Topic, #{}, <<1, 2, 3, 4, 5>>, [{qos, 1}]) + ). + +t_client_conn_bump_streams(Config) -> + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + {quic, Conn, _Stream} = proplists:get_value(socket, emqtt:info(C)), + ok = quicer:setopt(Conn, param_conn_settings, #{peer_unidi_stream_count => 20}). + +t_olp_true(Config) -> + meck:new(emqx_olp, [passthrough, no_history]), + ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + ok = meck:unload(emqx_olp). + +t_olp_reject(Config) -> + erlang:process_flag(trap_exit, true), + emqx_config:put_zone_conf(default, [overload_protection, enable], true), + meck:new(emqx_olp, [passthrough, no_history]), + ok = meck:expect(emqx_olp, is_overloaded, fun() -> true end), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + ?assertEqual( + {error, + {transport_down, #{ + error => 346, + status => + user_canceled + }}}, + emqtt:quic_connect(C) + ), + ok = meck:unload(emqx_olp), + emqx_config:put_zone_conf(default, [overload_protection, enable], false). + +t_conn_resume(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C0} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + + {ok, _} = emqtt:quic_connect(C0), + #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)), + emqtt:disconnect(C0), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5}, + {nst, NST} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + Cid = proplists:get_value(clientid, emqtt:info(C)), + ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]). + +t_conn_without_ctrl_stream(Config) -> + erlang:process_flag(trap_exit, true), + {ok, Conn} = quicer:connect( + {127, 0, 0, 1}, + ?config(port, Config), + [{alpn, ["mqtt"]}, {verify, none}], + 3000 + ), + receive + {quic, transport_shutdown, Conn, _} -> ok + end. + +t_data_stream_race_ctrl_stream(Config) -> + erlang:process_flag(trap_exit, true), + {ok, C0} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5} + | Config + ]), + {ok, _} = emqtt:quic_connect(C0), + #{nst := NST} = proplists:get_value(extra, emqtt:info(C0)), + emqtt:disconnect(C0), + {ok, C} = emqtt:start_link([ + {proto_ver, v5}, + {connect_timeout, 5}, + {nst, NST} + | Config + ]), + {ok, _} = emqtt:quic_connect(C), + Cid = proplists:get_value(clientid, emqtt:info(C)), + ct:pal("~p~n", [emqx_cm:get_chan_info(Cid)]). + +t_multi_streams_sub_0_rtt(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + #{}, + <<"qos 2 1">>, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := <<"qos 2 1">>, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + +t_multi_streams_sub_0_rtt_large_payload(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + Payload = binary:copy(<<"qos 2 1">>, 1600), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + ok = emqtt:publish_async( + C, + {new_data_stream, []}, + Topic, + #{}, + Payload, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := Payload, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + +%% @doc verify data stream can continue after 0-RTT handshake +t_multi_streams_sub_0_rtt_stream_data_cont(Config) -> + PubQos = ?config(pub_qos, Config), + SubQos = ?config(sub_qos, Config), + RecQos = calc_qos(PubQos, SubQos), + Topic = atom_to_binary(?FUNCTION_NAME), + Payload = binary:copy(<<"qos 2 1">>, 1600), + {ok, C0} = emqtt:start_link([{proto_ver, v5} | Config]), + {ok, _} = emqtt:quic_connect(C0), + {ok, _, [SubQos]} = emqtt:subscribe_via(C0, {new_data_stream, []}, #{}, [ + {Topic, [{qos, SubQos}]} + ]), + {ok, C} = emqtt:start_link([{proto_ver, v5} | Config]), + ok = emqtt:open_quic_connection(C), + ok = emqtt:quic_mqtt_connect(C), + {ok, PubVia} = emqtt:start_data_stream(C, []), + ok = emqtt:publish_async( + C, + PubVia, + Topic, + #{}, + Payload, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + {ok, _} = emqtt:quic_connect(C), + receive + {publish, #{ + client_pid := C0, + payload := Payload, + qos := RecQos, + topic := Topic + }} -> + ok; + Other -> + ct:fail("unexpected recv ~p", [Other]) + after 100 -> + ct:fail("not received") + end, + Payload2 = <<"2nd part", Payload/binary>>, + ok = emqtt:publish_async( + C, + PubVia, + Topic, + #{}, + Payload2, + [{qos, PubQos}], + infinity, + fun(_) -> ok end + ), + receive + {publish, #{ + client_pid := C0, + payload := Payload2, + qos := RecQos, + topic := Topic + }} -> + ok; + Other2 -> + ct:fail("unexpected recv ~p", [Other2]) + after 100 -> + ct:fail("not received") + end, + ok = emqtt:disconnect(C), + ok = emqtt:disconnect(C0). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- +send_and_recv_with(Sock) -> + {ok, {IP, _}} = emqtt_quic:sockname(Sock), + ?assert(lists:member(tuple_size(IP), [4, 8])), + ok = emqtt_quic:send(Sock, <<"ping">>), + emqtt_quic:setopts(Sock, [{active, false}]), + {ok, <<"pong">>} = emqtt_quic:recv(Sock, 0), + ok = emqtt_quic:setopts(Sock, [{active, 100}]), + {ok, Stats} = emqtt_quic:getstat(Sock, [send_cnt, recv_cnt]), + %% connection level counters, not stream level + [{send_cnt, _}, {recv_cnt, _}] = Stats. + +certfile(Config) -> + filename:join([test_dir(Config), "certs", "test.crt"]). + +keyfile(Config) -> + filename:join([test_dir(Config), "certs", "test.key"]). + +test_dir(Config) -> + filename:dirname(filename:dirname(proplists:get_value(data_dir, Config))). + +recv_pub(Count) -> + recv_pub(Count, [], 100). + +recv_pub(0, Acc, _Tout) -> + lists:reverse(Acc); +recv_pub(Count, Acc, Tout) -> + receive + {publish, _Prop} = Pub -> + recv_pub(Count - 1, [Pub | Acc], Tout) + after Tout -> + timeout + end. + +all_tc() -> + code:add_patha(filename:join(code:lib_dir(emqx), "ebin/")), + emqx_common_test_helpers:all(?MODULE). + +-spec calc_qos(0 | 1 | 2, 0 | 1 | 2) -> 0 | 1 | 2. +calc_qos(PubQos, SubQos) -> + if + PubQos > SubQos -> + SubQos; + SubQos > PubQos -> + PubQos; + true -> + PubQos + end. +-spec calc_pkt_id(0 | 1 | 2, non_neg_integer()) -> undefined | non_neg_integer(). +calc_pkt_id(0, _Id) -> + undefined; +calc_pkt_id(1, Id) -> + Id; +calc_pkt_id(2, Id) -> + Id. + +-spec start_emqx_quic(inet:port_number()) -> ok. +start_emqx_quic(UdpPort) -> + emqx_common_test_helpers:start_apps([]), + application:ensure_all_started(quicer), + emqx_common_test_helpers:ensure_quic_listener(?MODULE, UdpPort). + +-spec stop_emqx() -> ok. +stop_emqx() -> + emqx_common_test_helpers:stop_apps([]). + +%% select a random port picked by OS +-spec select_port() -> inet:port_number(). +select_port() -> + {ok, S} = gen_udp:open(0, [{reuseaddr, true}]), + {ok, {_, Port}} = inet:sockname(S), + gen_udp:close(S), + case os:type() of + {unix, darwin} -> + %% in MacOS, still get address_in_use after close port + timer:sleep(500); + _ -> + skip + end, + ct:pal("select port: ~p", [Port]), + Port. + +-spec via_stream({quic, quicer:connection_handle(), quicer:stream_handle()}) -> + quicer:stream_handle(). +via_stream({quic, _Conn, Stream}) -> + Stream. + +%% BUILD_WITHOUT_QUIC +-else. +-endif. diff --git a/apps/emqx/test/emqx_vm_SUITE.erl b/apps/emqx/test/emqx_vm_SUITE.erl index 35f37a41e..12f28ed28 100644 --- a/apps/emqx/test/emqx_vm_SUITE.erl +++ b/apps/emqx/test/emqx_vm_SUITE.erl @@ -50,12 +50,6 @@ t_systeminfo(_Config) -> ), ?assertEqual(undefined, emqx_vm:get_system_info(undefined)). -t_mem_info(_Config) -> - application:ensure_all_started(os_mon), - MemInfo = emqx_vm:mem_info(), - [{total_memory, _}, {used_memory, _}] = MemInfo, - application:stop(os_mon). - t_process_info(_Config) -> ProcessInfo = emqx_vm:get_process_info(), ?assertEqual(emqx_vm:process_info_keys(), [K || {K, _V} <- ProcessInfo]). diff --git a/apps/emqx_authn/src/emqx_authn.app.src b/apps/emqx_authn/src/emqx_authn.app.src index 0b5b0dedc..7fbdf787a 100644 --- a/apps/emqx_authn/src/emqx_authn.app.src +++ b/apps/emqx_authn/src/emqx_authn.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authn, [ {description, "EMQX Authentication"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {modules, []}, {registered, [emqx_authn_sup, emqx_authn_registry]}, {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]}, diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 0e6eeb6af..bedd169e2 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -74,7 +74,7 @@ query(_) -> undefined. query_timeout(type) -> emqx_schema:duration_ms(); query_timeout(desc) -> ?DESC(?FUNCTION_NAME); -query_timeout(default) -> "5s"; +query_timeout(default) -> <<"5s">>; query_timeout(_) -> undefined. %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index 3fea50147..f016db09a 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.13"}, + {vsn, "0.1.14"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 44ec0d28a..4adada182 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -108,7 +108,7 @@ authz_http_common_fields() -> })}, {request_timeout, mk_duration("Request timeout", #{ - required => false, default => "30s", desc => ?DESC(request_timeout) + required => false, default => <<"30s">>, desc => ?DESC(request_timeout) })} ] ++ maps:to_list( diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 5527c26d6..e68ab3a50 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -223,7 +223,7 @@ http_common_fields() -> {url, fun url/1}, {request_timeout, mk_duration("Request timeout", #{ - required => false, default => "30s", desc => ?DESC(request_timeout) + required => false, default => <<"30s">>, desc => ?DESC(request_timeout) })}, {body, ?HOCON(map(), #{required => false, desc => ?DESC(body)})} ] ++ diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index beac051a1..1d04dc362 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,9 +1,9 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.12"}, + {vsn, "0.1.13"}, {registered, []}, {mod, {emqx_conf_app, []}}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_ctl]}, {env, []}, {modules, []} ]}. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 9793e00d0..4862be5fe 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -145,7 +145,7 @@ fields("cluster") -> emqx_schema:duration(), #{ mapping => "ekka.cluster_autoclean", - default => "5m", + default => <<"5m">>, desc => ?DESC(cluster_autoclean), 'readOnly' => true } @@ -214,7 +214,7 @@ fields(cluster_mcast) -> sc( string(), #{ - default => "239.192.0.1", + default => <<"239.192.0.1">>, desc => ?DESC(cluster_mcast_addr), 'readOnly' => true } @@ -232,7 +232,7 @@ fields(cluster_mcast) -> sc( string(), #{ - default => "0.0.0.0", + default => <<"0.0.0.0">>, desc => ?DESC(cluster_mcast_iface), 'readOnly' => true } @@ -259,7 +259,7 @@ fields(cluster_mcast) -> sc( emqx_schema:bytesize(), #{ - default => "16KB", + default => <<"16KB">>, desc => ?DESC(cluster_mcast_sndbuf), 'readOnly' => true } @@ -268,7 +268,7 @@ fields(cluster_mcast) -> sc( emqx_schema:bytesize(), #{ - default => "16KB", + default => <<"16KB">>, desc => ?DESC(cluster_mcast_recbuf), 'readOnly' => true } @@ -277,7 +277,7 @@ fields(cluster_mcast) -> sc( emqx_schema:bytesize(), #{ - default => "32KB", + default => <<"32KB">>, desc => ?DESC(cluster_mcast_buffer), 'readOnly' => true } @@ -289,7 +289,7 @@ fields(cluster_dns) -> sc( string(), #{ - default => "localhost", + default => <<"localhost">>, desc => ?DESC(cluster_dns_name), 'readOnly' => true } @@ -318,7 +318,7 @@ fields(cluster_etcd) -> sc( string(), #{ - default => "emqxcl", + default => <<"emqxcl">>, desc => ?DESC(cluster_etcd_prefix), 'readOnly' => true } @@ -327,7 +327,7 @@ fields(cluster_etcd) -> sc( emqx_schema:duration(), #{ - default => "1m", + default => <<"1m">>, 'readOnly' => true, desc => ?DESC(cluster_etcd_node_ttl) } @@ -347,7 +347,7 @@ fields(cluster_k8s) -> sc( string(), #{ - default => "http://10.110.111.204:8080", + default => <<"http://10.110.111.204:8080">>, desc => ?DESC(cluster_k8s_apiserver), 'readOnly' => true } @@ -356,7 +356,7 @@ fields(cluster_k8s) -> sc( string(), #{ - default => "emqx", + default => <<"emqx">>, desc => ?DESC(cluster_k8s_service_name), 'readOnly' => true } @@ -374,7 +374,7 @@ fields(cluster_k8s) -> sc( string(), #{ - default => "default", + default => <<"default">>, desc => ?DESC(cluster_k8s_namespace), 'readOnly' => true } @@ -383,7 +383,7 @@ fields(cluster_k8s) -> sc( string(), #{ - default => "pod.local", + default => <<"pod.local">>, 'readOnly' => true, desc => ?DESC(cluster_k8s_suffix) } @@ -395,7 +395,7 @@ fields("node") -> sc( string(), #{ - default => "emqx@127.0.0.1", + default => <<"emqx@127.0.0.1">>, 'readOnly' => true, desc => ?DESC(node_name) } @@ -477,7 +477,7 @@ fields("node") -> hoconsc:union([disabled, emqx_schema:duration()]), #{ mapping => "emqx_machine.global_gc_interval", - default => "15m", + default => <<"15m">>, desc => ?DESC(node_global_gc_interval), 'readOnly' => true } @@ -497,7 +497,7 @@ fields("node") -> emqx_schema:duration_s(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP_SECONDS", - default => "30s", + default => <<"30s">>, desc => ?DESC(node_crash_dump_seconds), 'readOnly' => true } @@ -507,7 +507,7 @@ fields("node") -> emqx_schema:bytesize(), #{ mapping => "vm_args.-env ERL_CRASH_DUMP_BYTES", - default => "100MB", + default => <<"100MB">>, desc => ?DESC(node_crash_dump_bytes), 'readOnly' => true } @@ -517,7 +517,7 @@ fields("node") -> emqx_schema:duration_s(), #{ mapping => "vm_args.-kernel net_ticktime", - default => "2m", + default => <<"2m">>, 'readOnly' => true, desc => ?DESC(node_dist_net_ticktime) } @@ -624,7 +624,7 @@ fields("cluster_call") -> emqx_schema:duration(), #{ desc => ?DESC(cluster_call_retry_interval), - default => "1m" + default => <<"1m">> } )}, {"max_history", @@ -640,7 +640,7 @@ fields("cluster_call") -> emqx_schema:duration(), #{ desc => ?DESC(cluster_call_cleanup_interval), - default => "5m" + default => <<"5m">> } )} ]; @@ -712,7 +712,7 @@ fields("rpc") -> emqx_schema:duration(), #{ mapping => "gen_rpc.connect_timeout", - default => "5s", + default => <<"5s">>, desc => ?DESC(rpc_connect_timeout) } )}, @@ -745,7 +745,7 @@ fields("rpc") -> emqx_schema:duration(), #{ mapping => "gen_rpc.send_timeout", - default => "5s", + default => <<"5s">>, desc => ?DESC(rpc_send_timeout) } )}, @@ -754,7 +754,7 @@ fields("rpc") -> emqx_schema:duration(), #{ mapping => "gen_rpc.authentication_timeout", - default => "5s", + default => <<"5s">>, desc => ?DESC(rpc_authentication_timeout) } )}, @@ -763,7 +763,7 @@ fields("rpc") -> emqx_schema:duration(), #{ mapping => "gen_rpc.call_receive_timeout", - default => "15s", + default => <<"15s">>, desc => ?DESC(rpc_call_receive_timeout) } )}, @@ -772,7 +772,7 @@ fields("rpc") -> emqx_schema:duration_s(), #{ mapping => "gen_rpc.socket_keepalive_idle", - default => "15m", + default => <<"15m">>, desc => ?DESC(rpc_socket_keepalive_idle) } )}, @@ -781,7 +781,7 @@ fields("rpc") -> emqx_schema:duration_s(), #{ mapping => "gen_rpc.socket_keepalive_interval", - default => "75s", + default => <<"75s">>, desc => ?DESC(rpc_socket_keepalive_interval) } )}, @@ -799,7 +799,7 @@ fields("rpc") -> emqx_schema:bytesize(), #{ mapping => "gen_rpc.socket_sndbuf", - default => "1MB", + default => <<"1MB">>, desc => ?DESC(rpc_socket_sndbuf) } )}, @@ -808,7 +808,7 @@ fields("rpc") -> emqx_schema:bytesize(), #{ mapping => "gen_rpc.socket_recbuf", - default => "1MB", + default => <<"1MB">>, desc => ?DESC(rpc_socket_recbuf) } )}, @@ -817,7 +817,7 @@ fields("rpc") -> emqx_schema:bytesize(), #{ mapping => "gen_rpc.socket_buffer", - default => "1MB", + default => <<"1MB">>, desc => ?DESC(rpc_socket_buffer) } )}, @@ -861,7 +861,7 @@ fields("log_file_handler") -> sc( hoconsc:union([infinity, emqx_schema:bytesize()]), #{ - default => "50MB", + default => <<"50MB">>, desc => ?DESC("log_file_handler_max_size") } )} @@ -899,7 +899,7 @@ fields("log_overload_kill") -> sc( emqx_schema:bytesize(), #{ - default => "30MB", + default => <<"30MB">>, desc => ?DESC("log_overload_kill_mem_size") } )}, @@ -915,7 +915,7 @@ fields("log_overload_kill") -> sc( hoconsc:union([emqx_schema:duration_ms(), infinity]), #{ - default => "5s", + default => <<"5s">>, desc => ?DESC("log_overload_kill_restart_after") } )} @@ -942,7 +942,7 @@ fields("log_burst_limit") -> sc( emqx_schema:duration(), #{ - default => "1s", + default => <<"1s">>, desc => ?DESC("log_burst_limit_window_time") } )} @@ -1092,7 +1092,7 @@ log_handler_common_confs(Enable) -> sc( string(), #{ - default => "system", + default => <<"system">>, desc => ?DESC("common_handler_time_offset"), validator => fun validate_time_offset/1 } @@ -1169,9 +1169,9 @@ crash_dump_file_default() -> case os:getenv("RUNNER_LOG_DIR") of false -> %% testing, or running emqx app as deps - "log/erl_crash.dump"; + <<"log/erl_crash.dump">>; Dir -> - [filename:join([Dir, "erl_crash.dump"])] + unicode:characters_to_binary(filename:join([Dir, "erl_crash.dump"]), utf8) end. %% utils diff --git a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf index f9f79beb8..0de97d84b 100644 --- a/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf +++ b/apps/emqx_connector/i18n/emqx_connector_mqtt_schema.conf @@ -114,9 +114,13 @@ topic filters for remote.topic of ingress connections.""" desc { en: """If enable bridge mode. NOTE: This setting is only for MQTT protocol version older than 5.0, and the remote MQTT -broker MUST support this feature.""" +broker MUST support this feature. +If bridge_mode is set to true, the bridge will indicate to the remote broker that it is a bridge not an ordinary client. +This means that loop detection will be more effective and that retained messages will be propagated correctly.""" zh: """是否启用 Bridge Mode。 -注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。""" +注意：此设置只针对 MQTT 协议版本 < 5.0 有效，并且需要远程 MQTT Broker 支持 Bridge Mode。 +如果设置为 true ，桥接会告诉远端服务器当前连接是一个桥接而不是一个普通的客户端。 +这意味着消息回环检测会更加高效，并且远端服务器收到的保留消息的标志位会透传给本地。""" } label { en: "Bridge Mode" diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index aedc17c33..dfcf52902 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.14"}, + {vsn, "0.1.15"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 7c4a1fcf8..7d91e18b9 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -87,7 +87,7 @@ fields(config) -> sc( emqx_schema:duration_ms(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC("connect_timeout") } )}, diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl index 066e053d4..e06d6a9d7 100644 --- a/apps/emqx_connector/src/emqx_connector_mysql.erl +++ b/apps/emqx_connector/src/emqx_connector_mysql.erl @@ -391,22 +391,7 @@ proc_sql_params(TypeOrKey, SQLOrData, Params, #{params_tokens := ParamsTokens}) end. on_batch_insert(InstId, BatchReqs, InsertPart, Tokens, State) -> - JoinFun = fun - ([Msg]) -> - emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Msg); - ([H | T]) -> - lists:foldl( - fun(Msg, Acc) -> - Value = emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Msg), - <> - end, - emqx_plugin_libs_rule:proc_sql_param_str(Tokens, H), - T - ) - end, - {_, Msgs} = lists:unzip(BatchReqs), - JoinPart = JoinFun(Msgs), - SQL = <>, + SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens), on_sql_query(InstId, query, SQL, [], default_timeout, State). on_sql_query( diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 890227b9d..1fc994275 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -100,7 +100,11 @@ on_start( case maps:get(enable, SSL) of true -> [ - {ssl, required}, + %% note: this is converted to `required' in + %% `conn_opts/2', and there's a boolean guard + %% there; if this is set to `required' here, + %% that'll require changing `conn_opts/2''s guard. + {ssl, true}, {ssl_opts, emqx_tls_lib:to_client_opts(SSL)} ]; false -> diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl index 073b75ae8..e08804685 100644 --- a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -115,12 +115,12 @@ fields("server_configs") -> desc => ?DESC("clean_start") } )}, - {keepalive, mk_duration("MQTT Keepalive.", #{default => "300s"})}, + {keepalive, mk_duration("MQTT Keepalive.", #{default => <<"300s">>})}, {retry_interval, mk_duration( "Message retry interval. Delay for the MQTT bridge to retry sending the QoS1/QoS2 " "messages in case of ACK not received.", - #{default => "15s"} + #{default => <<"15s">>} )}, {max_inflight, mk( diff --git a/apps/emqx_ctl/README.md b/apps/emqx_ctl/README.md new file mode 100644 index 000000000..a91342606 --- /dev/null +++ b/apps/emqx_ctl/README.md @@ -0,0 +1,4 @@ +emqx_ctl +===== + +Backend module for `emqx_ctl` command. diff --git a/apps/emqx_ctl/rebar.config b/apps/emqx_ctl/rebar.config new file mode 100644 index 000000000..2656fd554 --- /dev/null +++ b/apps/emqx_ctl/rebar.config @@ -0,0 +1,2 @@ +{erl_opts, [debug_info]}. +{deps, []}. diff --git a/apps/emqx_ctl/src/emqx_ctl.app.src b/apps/emqx_ctl/src/emqx_ctl.app.src new file mode 100644 index 000000000..9de598a89 --- /dev/null +++ b/apps/emqx_ctl/src/emqx_ctl.app.src @@ -0,0 +1,15 @@ +{application, emqx_ctl, [ + {description, "Backend for emqx_ctl script"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_ctl_app, []}}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, + + {licenses, ["Apache-2.0"]}, + {links, []} +]}. diff --git a/apps/emqx/src/emqx_ctl.erl b/apps/emqx_ctl/src/emqx_ctl.erl similarity index 92% rename from apps/emqx/src/emqx_ctl.erl rename to apps/emqx_ctl/src/emqx_ctl.erl index 53eb5b888..a9aad0259 100644 --- a/apps/emqx/src/emqx_ctl.erl +++ b/apps/emqx_ctl/src/emqx_ctl.erl @@ -18,8 +18,7 @@ -behaviour(gen_server). --include("types.hrl"). --include("logger.hrl"). +-include_lib("kernel/include/logger.hrl"). -export([start_link/0, stop/0]). @@ -70,7 +69,7 @@ -define(SERVER, ?MODULE). -define(CMD_TAB, emqx_command). --spec start_link() -> startlink_ret(). +-spec start_link() -> {ok, pid()}. start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). @@ -103,7 +102,7 @@ cast(Msg) -> gen_server:cast(?SERVER, Msg). run_command([]) -> run_command(help, []); run_command([Cmd | Args]) -> - case emqx_misc:safe_to_existing_atom(Cmd) of + case safe_to_existing_atom(Cmd) of {ok, Cmd1} -> run_command(Cmd1, Args); _ -> @@ -122,7 +121,7 @@ run_command(Cmd, Args) when is_atom(Cmd) -> ok catch _:Reason:Stacktrace -> - ?SLOG(error, #{ + ?LOG_ERROR(#{ msg => "ctl_command_crashed", stacktrace => Stacktrace, reason => Reason @@ -220,7 +219,7 @@ format_usage(CmdParams, Desc, Width) -> %%-------------------------------------------------------------------- init([]) -> - ok = emqx_tables:new(?CMD_TAB, [protected, ordered_set]), + _ = ets:new(?CMD_TAB, [named_table, protected, ordered_set]), {ok, #state{seq = 0}}. handle_call({register_command, Cmd, MF, Opts}, _From, State = #state{seq = Seq}) -> @@ -229,23 +228,23 @@ handle_call({register_command, Cmd, MF, Opts}, _From, State = #state{seq = Seq}) ets:insert(?CMD_TAB, {{Seq, Cmd}, MF, Opts}), {reply, ok, next_seq(State)}; [[OriginSeq] | _] -> - ?SLOG(warning, #{msg => "CMD_overidden", cmd => Cmd, mf => MF}), + ?LOG_WARNING(#{msg => "CMD_overidden", cmd => Cmd, mf => MF}), true = ets:insert(?CMD_TAB, {{OriginSeq, Cmd}, MF, Opts}), {reply, ok, State} end; handle_call(Req, _From, State) -> - ?SLOG(error, #{msg => "unexpected_call", call => Req}), + ?LOG_ERROR(#{msg => "unexpected_call", call => Req}), {reply, ignored, State}. handle_cast({unregister_command, Cmd}, State) -> ets:match_delete(?CMD_TAB, {{'_', Cmd}, '_', '_'}), noreply(State); handle_cast(Msg, State) -> - ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}), + ?LOG_ERROR(#{msg => "unexpected_cast", cast => Msg}), noreply(State). handle_info(Info, State) -> - ?SLOG(error, #{msg => "unexpected_info", info => Info}), + ?LOG_ERROR(#{msg => "unexpected_info", info => Info}), noreply(State). terminate(_Reason, _State) -> @@ -272,3 +271,11 @@ zip_cmd([X | Xs], [Y | Ys]) -> [{X, Y} | zip_cmd(Xs, Ys)]; zip_cmd([X | Xs], []) -> [{X, ""} | zip_cmd(Xs, [])]; zip_cmd([], [Y | Ys]) -> [{"", Y} | zip_cmd([], Ys)]; zip_cmd([], []) -> []. + +safe_to_existing_atom(Str) -> + try + {ok, list_to_existing_atom(Str)} + catch + _:badarg -> + undefined + end. diff --git a/apps/emqx_ctl/src/emqx_ctl_app.erl b/apps/emqx_ctl/src/emqx_ctl_app.erl new file mode 100644 index 000000000..803ba90d3 --- /dev/null +++ b/apps/emqx_ctl/src/emqx_ctl_app.erl @@ -0,0 +1,18 @@ +%%%------------------------------------------------------------------- +%% @doc emqx_ctl public API +%% @end +%%%------------------------------------------------------------------- + +-module(emqx_ctl_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_ctl_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_ctl/src/emqx_ctl_sup.erl b/apps/emqx_ctl/src/emqx_ctl_sup.erl new file mode 100644 index 000000000..21086e424 --- /dev/null +++ b/apps/emqx_ctl/src/emqx_ctl_sup.erl @@ -0,0 +1,33 @@ +%%%------------------------------------------------------------------- +%% @doc emqx_ctl top level supervisor. +%% @end +%%%------------------------------------------------------------------- + +-module(emqx_ctl_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{ + strategy => one_for_all, + intensity => 0, + period => 1 + }, + ChildSpecs = [ + #{ + id => emqx_ctl, + start => {emqx_ctl, start_link, []}, + type => worker, + restart => permanent + } + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx/test/emqx_ctl_SUITE.erl b/apps/emqx_ctl/test/emqx_ctl_SUITE.erl similarity index 95% rename from apps/emqx/test/emqx_ctl_SUITE.erl rename to apps/emqx_ctl/test/emqx_ctl_SUITE.erl index 03f7b2148..46d9008e8 100644 --- a/apps/emqx/test/emqx_ctl_SUITE.erl +++ b/apps/emqx_ctl/test/emqx_ctl_SUITE.erl @@ -22,12 +22,10 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> [t_reg_unreg_command, t_run_commands, t_print, t_usage, t_unexpected]. init_per_suite(Config) -> - %% ensure stopped, this suite tests emqx_ctl process independently - application:stop(emqx), - ok = emqx_logger:set_log_level(emergency), + application:stop(emqx_ctl), Config. end_per_suite(_Config) -> diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index b6c95ca97..44260cbe1 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,10 +2,10 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, []}, {registered, [emqx_dashboard_sup]}, - {applications, [kernel, stdlib, mnesia, minirest, emqx]}, + {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, {mod, {emqx_dashboard_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl index f8b0918be..69f5bf34e 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl @@ -55,7 +55,7 @@ schema("/monitor/nodes/:node") -> parameters => [parameter_node(), parameter_latest()], responses => #{ 200 => hoconsc:mk(hoconsc:array(hoconsc:ref(sampler)), #{}), - 400 => emqx_dashboard_swagger:error_codes(['BAD_RPC'], <<"Bad RPC">>) + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Node not found">>) } } }; @@ -79,7 +79,7 @@ schema("/monitor_current/nodes/:node") -> parameters => [parameter_node()], responses => #{ 200 => hoconsc:mk(hoconsc:ref(sampler_current), #{}), - 400 => emqx_dashboard_swagger:error_codes(['BAD_RPC'], <<"Bad RPC">>) + 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Node not found">>) } } }. @@ -122,38 +122,31 @@ fields(sampler_current) -> monitor(get, #{query_string := QS, bindings := Bindings}) -> Latest = maps:get(<<"latest">>, QS, infinity), RawNode = maps:get(node, Bindings, all), - case emqx_misc:safe_to_existing_atom(RawNode, utf8) of - {ok, Node} -> - case emqx_dashboard_monitor:samplers(Node, Latest) of - {badrpc, {Node, Reason}} -> - Message = list_to_binary( - io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason]) - ), - {400, 'BAD_RPC', Message}; - Samplers -> - {200, Samplers} - end; - _ -> - Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])), - {400, 'BAD_RPC', Message} + with_node(RawNode, dashboard_samplers_fun(Latest)). + +dashboard_samplers_fun(Latest) -> + fun(NodeOrCluster) -> + case emqx_dashboard_monitor:samplers(NodeOrCluster, Latest) of + {badrpc, _} = Error -> Error; + Samplers -> {ok, Samplers} + end end. monitor_current(get, #{bindings := Bindings}) -> RawNode = maps:get(node, Bindings, all), + with_node(RawNode, fun emqx_dashboard_monitor:current_rate/1). + +with_node(RawNode, Fun) -> case emqx_misc:safe_to_existing_atom(RawNode, utf8) of {ok, NodeOrCluster} -> - case emqx_dashboard_monitor:current_rate(NodeOrCluster) of - {ok, CurrentRate} -> - {200, CurrentRate}; + case Fun(NodeOrCluster) of {badrpc, {Node, Reason}} -> - Message = list_to_binary( - io_lib:format("Bad node ~p, rpc failed ~p", [Node, Reason]) - ), - {400, 'BAD_RPC', Message} + {404, 'NOT_FOUND', io_lib:format("Node not found: ~p (~p)", [Node, Reason])}; + {ok, Result} -> + {200, Result} end; - {error, _} -> - Message = list_to_binary(io_lib:format("Bad node ~p", [RawNode])), - {400, 'BAD_RPC', Message} + _Error -> + {404, 'NOT_FOUND', io_lib:format("Node not found: ~p", [RawNode])} end. %% ------------------------------------------------------------------------------------------------- diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index ceb2415f8..7df661fb2 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -40,7 +40,7 @@ fields("dashboard") -> ?HOCON( emqx_schema:duration_s(), #{ - default => "10s", + default => <<"10s">>, desc => ?DESC(sample_interval), validator => fun validate_sample_interval/1 } @@ -49,7 +49,7 @@ fields("dashboard") -> ?HOCON( emqx_schema:duration(), #{ - default => "60m", + default => <<"60m">>, desc => ?DESC(token_expired_time) } )}, @@ -141,7 +141,7 @@ common_listener_fields() -> ?HOCON( emqx_schema:duration(), #{ - default => "10s", + default => <<"10s">>, desc => ?DESC(send_timeout) } )}, @@ -206,14 +206,14 @@ desc(_) -> undefined. default_username(type) -> binary(); -default_username(default) -> "admin"; +default_username(default) -> <<"admin">>; default_username(required) -> true; default_username(desc) -> ?DESC(default_username); default_username('readOnly') -> true; default_username(_) -> undefined. default_password(type) -> binary(); -default_password(default) -> "public"; +default_password(default) -> <<"public">>; default_password(required) -> true; default_password('readOnly') -> true; default_password(sensitive) -> true; diff --git a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl index 74c6d9cc1..bfbd9b973 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl @@ -22,8 +22,6 @@ -import(emqx_dashboard_SUITE, [auth_header_/0]). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx.hrl"). -include("emqx_dashboard.hrl"). -define(SERVER, "http://127.0.0.1:18083"). @@ -114,9 +112,9 @@ t_monitor_reset(_) -> ok. t_monitor_api_error(_) -> - {error, {400, #{<<"code">> := <<"BAD_RPC">>}}} = + {error, {404, #{<<"code">> := <<"NOT_FOUND">>}}} = request(["monitor", "nodes", 'emqx@127.0.0.2']), - {error, {400, #{<<"code">> := <<"BAD_RPC">>}}} = + {error, {404, #{<<"code">> := <<"NOT_FOUND">>}}} = request(["monitor_current", "nodes", 'emqx@127.0.0.2']), {error, {400, #{<<"code">> := <<"BAD_REQUEST">>}}} = request(["monitor"], "latest=0"), diff --git a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl index a797d3b43..c2266ad5b 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_remote_schema.erl @@ -32,8 +32,8 @@ fields("root") -> )}, {default_username, fun default_username/1}, {default_password, fun default_password/1}, - {sample_interval, mk(emqx_schema:duration_s(), #{default => "10s"})}, - {token_expired_time, mk(emqx_schema:duration(), #{default => "30m"})} + {sample_interval, mk(emqx_schema:duration_s(), #{default => <<"10s">>})}, + {token_expired_time, mk(emqx_schema:duration(), #{default => <<"30m">>})} ]; fields("ref1") -> [ @@ -52,7 +52,7 @@ fields("ref3") -> ]. default_username(type) -> string(); -default_username(default) -> "admin"; +default_username(default) -> <<"admin">>; default_username(required) -> true; default_username(_) -> undefined. diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index d17725e80..979d01c77 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -790,7 +790,7 @@ to_schema(Body) -> fields(good_ref) -> [ - {'webhook-host', mk(emqx_schema:ip_port(), #{default => "127.0.0.1:80"})}, + {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})}, {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})}, {tag, mk(binary(), #{desc => <<"tag">>})} ]; diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index 346f4ef71..c9cfba254 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -689,7 +689,7 @@ to_schema(Object) -> fields(good_ref) -> [ - {'webhook-host', mk(emqx_schema:ip_port(), #{default => "127.0.0.1:80"})}, + {'webhook-host', mk(emqx_schema:ip_port(), #{default => <<"127.0.0.1:80">>})}, {log_dir, mk(emqx_schema:file(), #{example => "var/log/emqx"})}, {tag, mk(binary(), #{desc => <<"tag">>})} ]; diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index d81819c98..04e0a57db 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.9"}, + {vsn, "5.0.10"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_api.erl b/apps/emqx_exhook/src/emqx_exhook_api.erl index 4d7de2866..bcfc68269 100644 --- a/apps/emqx_exhook/src/emqx_exhook_api.erl +++ b/apps/emqx_exhook/src/emqx_exhook_api.erl @@ -229,9 +229,9 @@ server_conf_schema() -> name => "default", enable => true, url => <<"http://127.0.0.1:8081">>, - request_timeout => "5s", + request_timeout => <<"5s">>, failed_action => deny, - auto_reconnect => "60s", + auto_reconnect => <<"60s">>, pool_size => 8, ssl => SSL } diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index ce79dddac..07373288d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -63,7 +63,7 @@ fields(server) -> })}, {request_timeout, ?HOCON(emqx_schema:duration(), #{ - default => "5s", + default => <<"5s">>, desc => ?DESC(request_timeout) })}, {failed_action, failed_action()}, @@ -74,7 +74,7 @@ fields(server) -> })}, {auto_reconnect, ?HOCON(hoconsc:union([false, emqx_schema:duration()]), #{ - default => "60s", + default => <<"60s">>, desc => ?DESC(auto_reconnect) })}, {pool_size, diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index c5dd76f19..787af7429 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,10 +1,10 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.11"}, + {vsn, "0.1.12"}, {registered, []}, {mod, {emqx_gateway_app, []}}, - {applications, [kernel, stdlib, grpc, emqx, emqx_authn]}, + {applications, [kernel, stdlib, grpc, emqx, emqx_authn, emqx_ctl]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index ef1c4c386..b30de3a3e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -19,7 +19,6 @@ -include("emqx_gateway_http.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx/include/logger.hrl"). -behaviour(minirest_api). @@ -464,7 +463,12 @@ schema("/gateways/:name/clients") -> summary => <<"List Gateway's Clients">>, parameters => params_client_query(), responses => - ?STANDARD_RESP(#{200 => schema_client_list()}) + ?STANDARD_RESP(#{ + 200 => [ + {data, schema_client_list()}, + {meta, mk(hoconsc:ref(emqx_dashboard_swagger, meta), #{})} + ] + }) } }; schema("/gateways/:name/clients/:clientid") -> diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 4ea845ea1..2034a40eb 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -267,7 +267,7 @@ fields(lwm2m) -> sc( duration(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC(lwm2m_lifetime_min) } )}, @@ -275,7 +275,7 @@ fields(lwm2m) -> sc( duration(), #{ - default => "86400s", + default => <<"86400s">>, desc => ?DESC(lwm2m_lifetime_max) } )}, @@ -283,7 +283,7 @@ fields(lwm2m) -> sc( duration_s(), #{ - default => "22s", + default => <<"22s">>, desc => ?DESC(lwm2m_qmode_time_window) } )}, @@ -624,7 +624,7 @@ mountpoint(Default) -> sc( binary(), #{ - default => Default, + default => iolist_to_binary(Default), desc => ?DESC(gateway_common_mountpoint) } ). @@ -707,7 +707,7 @@ proxy_protocol_opts() -> sc( duration(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC(tcp_listener_proxy_protocol_timeout) } )} diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index 4acb3cb84..7c62b0685 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -6,7 +6,7 @@ {vsn, "0.2.0"}, {modules, []}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_ctl]}, {mod, {emqx_machine_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index ec0aff55b..9dc3fdc54 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -29,6 +29,7 @@ %% @doc EMQX boot entrypoint. start() -> + emqx_mgmt_cli:load(), case os:type() of {win32, nt} -> ok; diff --git a/apps/emqx_management/include/emqx_mgmt.hrl b/apps/emqx_management/include/emqx_mgmt.hrl index b68a9a634..7f6b5a675 100644 --- a/apps/emqx_management/include/emqx_mgmt.hrl +++ b/apps/emqx_management/include/emqx_mgmt.hrl @@ -16,4 +16,4 @@ -define(MANAGEMENT_SHARD, emqx_management_shard). --define(MAX_ROW_LIMIT, 100). +-define(DEFAULT_ROW_LIMIT, 100). diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index fdd2af9b2..08de7b670 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,10 +2,10 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {registered, [emqx_management_sup]}, - {applications, [kernel, stdlib, emqx_plugins, minirest, emqx]}, + {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, {mod, {emqx_mgmt_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 814b39cdc..efa5a03bd 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -21,8 +21,6 @@ -elvis([{elvis_style, god_modules, disable}]). -include_lib("stdlib/include/qlc.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). %% Nodes and Brokers API -export([ @@ -71,8 +69,6 @@ list_subscriptions/1, list_subscriptions_via_topic/2, list_subscriptions_via_topic/3, - lookup_subscriptions/1, - lookup_subscriptions/2, do_list_subscriptions/0 ]). @@ -104,9 +100,10 @@ ]). %% Common Table API --export([max_row_limit/0]). - --define(APP, emqx_management). +-export([ + default_row_limit/0, + vm_stats/0 +]). -elvis([{elvis_style, god_modules, disable}]). @@ -159,7 +156,24 @@ node_info(Nodes) -> emqx_rpc:unwrap_erpc(emqx_management_proto_v3:node_info(Nodes)). stopped_node_info(Node) -> - #{name => Node, node_status => 'stopped'}. + {Node, #{node => Node, node_status => 'stopped'}}. + +vm_stats() -> + Idle = + case cpu_sup:util([detailed]) of + %% Not support for Windows + {_, 0, 0, _} -> 0; + {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0) + end, + RunQueue = erlang:statistics(run_queue), + {MemUsedRatio, MemTotal} = get_sys_memory(), + [ + {run_queue, RunQueue}, + {cpu_idle, Idle}, + {cpu_use, 100 - Idle}, + {total_memory, MemTotal}, + {used_memory, erlang:round(MemTotal * MemUsedRatio)} + ]. %%-------------------------------------------------------------------- %% Brokers @@ -174,8 +188,13 @@ lookup_broker(Node) -> Broker. broker_info() -> - Info = maps:from_list([{K, iolist_to_binary(V)} || {K, V} <- emqx_sys:info()]), - Info#{node => node(), otp_release => otp_rel(), node_status => 'Running'}. + Info = lists:foldl(fun convert_broker_info/2, #{}, emqx_sys:info()), + Info#{node => node(), otp_release => otp_rel(), node_status => 'running'}. + +convert_broker_info({uptime, Uptime}, M) -> + M#{uptime => emqx_datetime:human_readable_duration_string(Uptime)}; +convert_broker_info({K, V}, M) -> + M#{K => iolist_to_binary(V)}. broker_info(Nodes) -> emqx_rpc:unwrap_erpc(emqx_management_proto_v3:broker_info(Nodes)). @@ -245,7 +264,7 @@ lookup_client({username, Username}, FormatFun) -> || Node <- mria_mnesia:running_nodes() ]). -lookup_client(Node, Key, {M, F}) -> +lookup_client(Node, Key, FormatFun) -> case unwrap_rpc(emqx_cm_proto_v1:lookup_client(Node, Key)) of {error, Err} -> {error, Err}; @@ -253,18 +272,23 @@ lookup_client(Node, Key, {M, F}) -> lists:map( fun({Chan, Info0, Stats}) -> Info = Info0#{node => Node}, - M:F({Chan, Info, Stats}) + maybe_format(FormatFun, {Chan, Info, Stats}) end, L ) end. -kickout_client({ClientID, FormatFun}) -> - case lookup_client({clientid, ClientID}, FormatFun) of +maybe_format(undefined, A) -> + A; +maybe_format({M, F}, A) -> + M:F(A). + +kickout_client(ClientId) -> + case lookup_client({clientid, ClientId}, undefined) of [] -> {error, not_found}; _ -> - Results = [kickout_client(Node, ClientID) || Node <- mria_mnesia:running_nodes()], + Results = [kickout_client(Node, ClientId) || Node <- mria_mnesia:running_nodes()], check_results(Results) end. @@ -275,17 +299,22 @@ list_authz_cache(ClientId) -> call_client(ClientId, list_authz_cache). list_client_subscriptions(ClientId) -> - Results = [client_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()], - Filter = - fun - ({error, _}) -> - false; - ({_Node, List}) -> - erlang:is_list(List) andalso 0 < erlang:length(List) - end, - case lists:filter(Filter, Results) of - [] -> []; - [Result | _] -> Result + case lookup_client({clientid, ClientId}, undefined) of + [] -> + {error, not_found}; + _ -> + Results = [client_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()], + Filter = + fun + ({error, _}) -> + false; + ({_Node, List}) -> + erlang:is_list(List) andalso 0 < erlang:length(List) + end, + case lists:filter(Filter, Results) of + [] -> []; + [Result | _] -> Result + end end. client_subscriptions(Node, ClientId) -> @@ -368,17 +397,11 @@ call_client(Node, ClientId, Req) -> %% Subscriptions %%-------------------------------------------------------------------- --spec do_list_subscriptions() -> [map()]. +-spec do_list_subscriptions() -> no_return(). do_list_subscriptions() -> - case check_row_limit([mqtt_subproperty]) of - false -> - throw(max_row_limit); - ok -> - [ - #{topic => Topic, clientid => ClientId, options => Options} - || {{Topic, ClientId}, Options} <- ets:tab2list(mqtt_subproperty) - ] - end. + %% [FIXME] Add function to `emqx_broker` that returns list of subscriptions + %% and either redirect from here or bpapi directly (EMQX-8993). + throw(not_implemented). list_subscriptions(Node) -> unwrap_rpc(emqx_management_proto_v3:list_subscriptions(Node)). @@ -395,12 +418,6 @@ list_subscriptions_via_topic(Node, Topic, _FormatFun = {M, F}) -> Result -> M:F(Result) end. -lookup_subscriptions(ClientId) -> - lists:append([lookup_subscriptions(Node, ClientId) || Node <- mria_mnesia:running_nodes()]). - -lookup_subscriptions(Node, ClientId) -> - unwrap_rpc(emqx_broker_proto_v1:list_client_subscriptions(Node, ClientId)). - %%-------------------------------------------------------------------- %% PubSub %%-------------------------------------------------------------------- @@ -536,24 +553,11 @@ unwrap_rpc(Res) -> otp_rel() -> iolist_to_binary([emqx_vm:get_otp_version(), "/", erlang:system_info(version)]). -check_row_limit(Tables) -> - check_row_limit(Tables, max_row_limit()). - -check_row_limit([], _Limit) -> - ok; -check_row_limit([Tab | Tables], Limit) -> - case table_size(Tab) > Limit of - true -> false; - false -> check_row_limit(Tables, Limit) - end. - check_results(Results) -> case lists:any(fun(Item) -> Item =:= ok end, Results) of true -> ok; false -> unwrap_rpc(lists:last(Results)) end. -max_row_limit() -> - ?MAX_ROW_LIMIT. - -table_size(Tab) -> ets:info(Tab, size). +default_row_limit() -> + ?DEFAULT_ROW_LIMIT. diff --git a/apps/emqx_management/src/emqx_mgmt_api.erl b/apps/emqx_management/src/emqx_mgmt_api.erl index e46047521..a0a40533d 100644 --- a/apps/emqx_management/src/emqx_mgmt_api.erl +++ b/apps/emqx_management/src/emqx_mgmt_api.erl @@ -23,8 +23,7 @@ -define(LONG_QUERY_TIMEOUT, 50000). -export([ - paginate/3, - paginate/4 + paginate/3 ]). %% first_next query APIs @@ -34,6 +33,10 @@ b2i/1 ]). +-ifdef(TEST). +-export([paginate_test_format/1]). +-endif. + -export_type([ match_spec_and_filter/0 ]). @@ -58,14 +61,14 @@ -export([do_query/2, apply_total_query/1]). -paginate(Tables, Params, {Module, FormatFun}) -> - Qh = query_handle(Tables), - Count = count(Tables), - do_paginate(Qh, Count, Params, {Module, FormatFun}). - -paginate(Tables, MatchSpec, Params, {Module, FormatFun}) -> - Qh = query_handle(Tables, MatchSpec), - Count = count(Tables, MatchSpec), +-spec paginate(atom(), map(), {atom(), atom()}) -> + #{ + meta => #{page => pos_integer(), limit => pos_integer(), count => pos_integer()}, + data => list(term()) + }. +paginate(Table, Params, {Module, FormatFun}) -> + Qh = query_handle(Table), + Count = count(Table), do_paginate(Qh, Count, Params, {Module, FormatFun}). do_paginate(Qh, Count, Params, {Module, FormatFun}) -> @@ -86,57 +89,17 @@ do_paginate(Qh, Count, Params, {Module, FormatFun}) -> data => [erlang:apply(Module, FormatFun, [Row]) || Row <- Rows] }. -query_handle(Table) when is_atom(Table) -> - qlc:q([R || R <- ets:table(Table)]); -query_handle({Table, Opts}) when is_atom(Table) -> - qlc:q([R || R <- ets:table(Table, Opts)]); -query_handle([Table]) when is_atom(Table) -> - qlc:q([R || R <- ets:table(Table)]); -query_handle([{Table, Opts}]) when is_atom(Table) -> - qlc:q([R || R <- ets:table(Table, Opts)]); -query_handle(Tables) -> - % - qlc:append([query_handle(T) || T <- Tables]). +query_handle(Table) -> + qlc:q([R || R <- ets:table(Table)]). -query_handle(Table, MatchSpec) when is_atom(Table) -> - Options = {traverse, {select, MatchSpec}}, - qlc:q([R || R <- ets:table(Table, Options)]); -query_handle([Table], MatchSpec) when is_atom(Table) -> - Options = {traverse, {select, MatchSpec}}, - qlc:q([R || R <- ets:table(Table, Options)]); -query_handle(Tables, MatchSpec) -> - Options = {traverse, {select, MatchSpec}}, - qlc:append([qlc:q([E || E <- ets:table(T, Options)]) || T <- Tables]). +count(Table) -> + ets:info(Table, size). -count(Table) when is_atom(Table) -> - ets:info(Table, size); -count({Table, _}) when is_atom(Table) -> - ets:info(Table, size); -count([Table]) when is_atom(Table) -> - ets:info(Table, size); -count([{Table, _}]) when is_atom(Table) -> - ets:info(Table, size); -count(Tables) -> - lists:sum([count(T) || T <- Tables]). - -count(Table, MatchSpec) when is_atom(Table) -> - [{MatchPattern, Where, _Re}] = MatchSpec, - NMatchSpec = [{MatchPattern, Where, [true]}], - ets:select_count(Table, NMatchSpec); -count([Table], MatchSpec) when is_atom(Table) -> - count(Table, MatchSpec); -count(Tables, MatchSpec) -> - lists:sum([count(T, MatchSpec) || T <- Tables]). - -page(Params) when is_map(Params) -> - maps:get(<<"page">>, Params, 1); page(Params) -> - proplists:get_value(<<"page">>, Params, <<"1">>). + maps:get(<<"page">>, Params, 1). limit(Params) when is_map(Params) -> - maps:get(<<"limit">>, Params, emqx_mgmt:max_row_limit()); -limit(Params) -> - proplists:get_value(<<"limit">>, Params, emqx_mgmt:max_row_limit()). + maps:get(<<"limit">>, Params, emqx_mgmt:default_row_limit()). %%-------------------------------------------------------------------- %% Node Query @@ -210,8 +173,6 @@ cluster_query(Tab, QString, QSchema, MsFun, FmtFun) -> end. %% @private -do_cluster_query([], QueryState, ResultAcc) -> - finalize_query(ResultAcc, mark_complete(QueryState)); do_cluster_query( [Node | Tail] = Nodes, QueryState, @@ -605,7 +566,7 @@ to_type(V, TargetType) -> to_type_(V, atom) -> to_atom(V); to_type_(V, integer) -> to_integer(V); to_type_(V, timestamp) -> to_timestamp(V); -to_type_(V, ip) -> aton(V); +to_type_(V, ip) -> to_ip(V); to_type_(V, ip_port) -> to_ip_port(V); to_type_(V, _) -> V. @@ -624,14 +585,16 @@ to_timestamp(I) when is_integer(I) -> to_timestamp(B) when is_binary(B) -> binary_to_integer(B). -aton(B) when is_binary(B) -> - list_to_tuple([binary_to_integer(T) || T <- re:split(B, "[.]")]). +to_ip(IP0) when is_binary(IP0) -> + ensure_ok(inet:parse_address(binary_to_list(IP0))). to_ip_port(IPAddress) -> - [IP0, Port0] = string:tokens(binary_to_list(IPAddress), ":"), - {ok, IP} = inet:parse_address(IP0), - Port = list_to_integer(Port0), - {IP, Port}. + ensure_ok(emqx_schema:to_ip_port(IPAddress)). + +ensure_ok({ok, V}) -> + V; +ensure_ok({error, _R} = E) -> + throw(E). b2i(Bin) when is_binary(Bin) -> binary_to_integer(Bin); @@ -645,40 +608,115 @@ b2i(Any) -> -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -params2qs_test() -> +params2qs_test_() -> QSchema = [ {<<"str">>, binary}, {<<"int">>, integer}, + {<<"binatom">>, atom}, {<<"atom">>, atom}, {<<"ts">>, timestamp}, {<<"gte_range">>, integer}, {<<"lte_range">>, integer}, {<<"like_fuzzy">>, binary}, - {<<"match_topic">>, binary} + {<<"match_topic">>, binary}, + {<<"ip">>, ip}, + {<<"ip_port">>, ip_port} ], QString = [ {<<"str">>, <<"abc">>}, {<<"int">>, <<"123">>}, - {<<"atom">>, <<"connected">>}, + {<<"binatom">>, <<"connected">>}, + {<<"atom">>, ok}, {<<"ts">>, <<"156000">>}, {<<"gte_range">>, <<"1">>}, {<<"lte_range">>, <<"5">>}, {<<"like_fuzzy">>, <<"user">>}, - {<<"match_topic">>, <<"t/#">>} + {<<"match_topic">>, <<"t/#">>}, + {<<"ip">>, <<"127.0.0.1">>}, + {<<"ip_port">>, <<"127.0.0.1:8888">>} ], ExpectedQs = [ {str, '=:=', <<"abc">>}, {int, '=:=', 123}, - {atom, '=:=', connected}, + {binatom, '=:=', connected}, + {atom, '=:=', ok}, {ts, '=:=', 156000}, - {range, '>=', 1, '=<', 5} + {range, '>=', 1, '=<', 5}, + {ip, '=:=', {127, 0, 0, 1}}, + {ip_port, '=:=', {{127, 0, 0, 1}, 8888}} ], FuzzyNQString = [ {fuzzy, like, <<"user">>}, {topic, match, <<"t/#">>} ], - ?assertEqual({7, {ExpectedQs, FuzzyNQString}}, parse_qstring(QString, QSchema)), - {0, {[], []}} = parse_qstring([{not_a_predefined_params, val}], QSchema). + [ + ?_assertEqual({10, {ExpectedQs, FuzzyNQString}}, parse_qstring(QString, QSchema)), + ?_assertEqual({0, {[], []}}, parse_qstring([{not_a_predefined_params, val}], QSchema)), + ?_assertEqual( + {1, {[{ip, '=:=', {0, 0, 0, 0, 0, 0, 0, 1}}], []}}, + parse_qstring([{<<"ip">>, <<"::1">>}], QSchema) + ), + ?_assertEqual( + {1, {[{ip_port, '=:=', {{0, 0, 0, 0, 0, 0, 0, 1}, 8888}}], []}}, + parse_qstring([{<<"ip_port">>, <<"::1:8888">>}], QSchema) + ), + ?_assertThrow( + {bad_value_type, {<<"ip">>, ip, <<"helloworld">>}}, + parse_qstring([{<<"ip">>, <<"helloworld">>}], QSchema) + ), + ?_assertThrow( + {bad_value_type, {<<"ip_port">>, ip_port, <<"127.0.0.1">>}}, + parse_qstring([{<<"ip_port">>, <<"127.0.0.1">>}], QSchema) + ), + ?_assertThrow( + {bad_value_type, {<<"ip_port">>, ip_port, <<"helloworld:abcd">>}}, + parse_qstring([{<<"ip_port">>, <<"helloworld:abcd">>}], QSchema) + ) + ]. +paginate_test_format(Row) -> + Row. + +paginate_test_() -> + _ = ets:new(?MODULE, [named_table]), + Size = 1000, + MyLimit = 10, + ets:insert(?MODULE, [{I, foo} || I <- lists:seq(1, Size)]), + DefaultLimit = emqx_mgmt:default_row_limit(), + NoParamsResult = paginate(?MODULE, #{}, {?MODULE, paginate_test_format}), + PaginateResults = [ + paginate( + ?MODULE, #{<<"page">> => I, <<"limit">> => MyLimit}, {?MODULE, paginate_test_format} + ) + || I <- lists:seq(1, floor(Size / MyLimit)) + ], + [ + ?_assertMatch( + #{meta := #{count := Size, page := 1, limit := DefaultLimit}}, NoParamsResult + ), + ?_assertEqual(DefaultLimit, length(maps:get(data, NoParamsResult))), + ?_assertEqual( + #{data => [], meta => #{count => Size, limit => DefaultLimit, page => 100}}, + paginate(?MODULE, #{<<"page">> => <<"100">>}, {?MODULE, paginate_test_format}) + ) + ] ++ assert_paginate_results(PaginateResults, Size, MyLimit). + +assert_paginate_results(Results, Size, Limit) -> + AllData = lists:flatten([Data || #{data := Data} <- Results]), + [ + begin + Result = lists:nth(I, Results), + [ + ?_assertMatch(#{meta := #{count := Size, limit := Limit, page := I}}, Result), + ?_assertEqual(Limit, length(maps:get(data, Result))) + ] + end + || I <- lists:seq(1, floor(Size / Limit)) + ] ++ + [ + ?_assertEqual(floor(Size / Limit), length(Results)), + ?_assertEqual(Size, length(AllData)), + ?_assertEqual(Size, sets:size(sets:from_list(AllData))) + ]. -endif. diff --git a/apps/emqx_management/src/emqx_mgmt_api_clients.erl b/apps/emqx_management/src/emqx_mgmt_api_clients.erl index 7c45206fd..cac3edaed 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_clients.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_clients.erl @@ -76,9 +76,10 @@ -define(FORMAT_FUN, {?MODULE, format_channel_info}). --define(CLIENT_ID_NOT_FOUND, - <<"{\"code\": \"RESOURCE_NOT_FOUND\", \"reason\": \"Client id not found\"}">> -). +-define(CLIENTID_NOT_FOUND, #{ + code => 'CLIENTID_NOT_FOUND', + message => <<"Client ID not found">> +}). api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}). @@ -219,7 +220,7 @@ schema("/clients/:clientid") -> responses => #{ 200 => hoconsc:mk(hoconsc:ref(?MODULE, client), #{}), 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } }, @@ -232,7 +233,7 @@ schema("/clients/:clientid") -> responses => #{ 204 => <<"Kick out client successfully">>, 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } } @@ -247,7 +248,7 @@ schema("/clients/:clientid/authorization/cache") -> responses => #{ 200 => hoconsc:mk(hoconsc:ref(?MODULE, authz_cache), #{}), 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } }, @@ -256,9 +257,9 @@ schema("/clients/:clientid/authorization/cache") -> tags => ?TAGS, parameters => [{clientid, hoconsc:mk(binary(), #{in => path})}], responses => #{ - 204 => <<"Kick out client successfully">>, + 204 => <<"Clean client authz cache successfully">>, 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } } @@ -275,7 +276,7 @@ schema("/clients/:clientid/subscriptions") -> hoconsc:array(hoconsc:ref(emqx_mgmt_api_subscriptions, subscription)), #{} ), 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } } @@ -291,7 +292,7 @@ schema("/clients/:clientid/subscribe") -> responses => #{ 200 => hoconsc:ref(emqx_mgmt_api_subscriptions, subscription), 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } } @@ -307,7 +308,7 @@ schema("/clients/:clientid/subscribe/bulk") -> responses => #{ 200 => hoconsc:array(hoconsc:ref(emqx_mgmt_api_subscriptions, subscription)), 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } } @@ -323,7 +324,7 @@ schema("/clients/:clientid/unsubscribe") -> responses => #{ 204 => <<"Unsubscribe OK">>, 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } } @@ -339,7 +340,7 @@ schema("/clients/:clientid/unsubscribe/bulk") -> responses => #{ 204 => <<"Unsubscribe OK">>, 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } } @@ -355,7 +356,7 @@ schema("/clients/:clientid/keepalive") -> responses => #{ 200 => hoconsc:mk(hoconsc:ref(?MODULE, client), #{}), 404 => emqx_dashboard_swagger:error_codes( - ['CLIENTID_NOT_FOUND'], <<"Client id not found">> + ['CLIENTID_NOT_FOUND'], <<"Client ID not found">> ) } } @@ -597,6 +598,8 @@ unsubscribe_batch(post, #{bindings := #{clientid := ClientID}, body := TopicInfo subscriptions(get, #{bindings := #{clientid := ClientID}}) -> case emqx_mgmt:list_client_subscriptions(ClientID) of + {error, not_found} -> + {404, ?CLIENTID_NOT_FOUND}; [] -> {200, []}; {Node, Subs} -> @@ -621,7 +624,7 @@ set_keepalive(put, #{bindings := #{clientid := ClientID}, body := Body}) -> {ok, Interval} -> case emqx_mgmt:set_keepalive(emqx_mgmt_util:urldecode(ClientID), Interval) of ok -> lookup(#{clientid => ClientID}); - {error, not_found} -> {404, ?CLIENT_ID_NOT_FOUND}; + {error, not_found} -> {404, ?CLIENTID_NOT_FOUND}; {error, Reason} -> {400, #{code => 'PARAMS_ERROR', message => Reason}} end end. @@ -669,15 +672,15 @@ list_clients(QString) -> lookup(#{clientid := ClientID}) -> case emqx_mgmt:lookup_client({clientid, ClientID}, ?FORMAT_FUN) of [] -> - {404, ?CLIENT_ID_NOT_FOUND}; + {404, ?CLIENTID_NOT_FOUND}; ClientInfo -> {200, hd(ClientInfo)} end. kickout(#{clientid := ClientID}) -> - case emqx_mgmt:kickout_client({ClientID, ?FORMAT_FUN}) of + case emqx_mgmt:kickout_client(ClientID) of {error, not_found} -> - {404, ?CLIENT_ID_NOT_FOUND}; + {404, ?CLIENTID_NOT_FOUND}; _ -> {204} end. @@ -685,7 +688,7 @@ kickout(#{clientid := ClientID}) -> get_authz_cache(#{clientid := ClientID}) -> case emqx_mgmt:list_authz_cache(ClientID) of {error, not_found} -> - {404, ?CLIENT_ID_NOT_FOUND}; + {404, ?CLIENTID_NOT_FOUND}; {error, Reason} -> Message = list_to_binary(io_lib:format("~p", [Reason])), {500, #{code => <<"UNKNOW_ERROR">>, message => Message}}; @@ -699,7 +702,7 @@ clean_authz_cache(#{clientid := ClientID}) -> ok -> {204}; {error, not_found} -> - {404, ?CLIENT_ID_NOT_FOUND}; + {404, ?CLIENTID_NOT_FOUND}; {error, Reason} -> Message = list_to_binary(io_lib:format("~p", [Reason])), {500, #{code => <<"UNKNOW_ERROR">>, message => Message}} @@ -709,7 +712,7 @@ subscribe(#{clientid := ClientID, topic := Topic} = Sub) -> Opts = maps:with([qos, nl, rap, rh], Sub), case do_subscribe(ClientID, Topic, Opts) of {error, channel_not_found} -> - {404, ?CLIENT_ID_NOT_FOUND}; + {404, ?CLIENTID_NOT_FOUND}; {error, Reason} -> Message = list_to_binary(io_lib:format("~p", [Reason])), {500, #{code => <<"UNKNOW_ERROR">>, message => Message}}; @@ -723,7 +726,7 @@ subscribe_batch(#{clientid := ClientID, topics := Topics}) -> %% has returned. So if one want to subscribe topics in this hook, it will fail. case ets:lookup(emqx_channel, ClientID) of [] -> - {404, ?CLIENT_ID_NOT_FOUND}; + {404, ?CLIENTID_NOT_FOUND}; _ -> ArgList = [ [ClientID, Topic, maps:with([qos, nl, rap, rh], Sub)] @@ -735,7 +738,7 @@ subscribe_batch(#{clientid := ClientID, topics := Topics}) -> unsubscribe(#{clientid := ClientID, topic := Topic}) -> case do_unsubscribe(ClientID, Topic) of {error, channel_not_found} -> - {404, ?CLIENT_ID_NOT_FOUND}; + {404, ?CLIENTID_NOT_FOUND}; {unsubscribe, [{Topic, #{}}]} -> {204} end. @@ -745,8 +748,8 @@ unsubscribe_batch(#{clientid := ClientID, topics := Topics}) -> {200, _} -> _ = emqx_mgmt:unsubscribe_batch(ClientID, Topics), {204}; - {404, ?CLIENT_ID_NOT_FOUND} -> - {404, ?CLIENT_ID_NOT_FOUND} + {404, NotFound} -> + {404, NotFound} end. %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/src/emqx_mgmt_api_trace.erl b/apps/emqx_management/src/emqx_mgmt_api_trace.erl index cc4a905a4..38ce9dcf2 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_trace.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_trace.erl @@ -47,9 +47,11 @@ get_trace_size/0 ]). +-define(MAX_SINT32, 2147483647). + -define(TO_BIN(_B_), iolist_to_binary(_B_)). -define(NOT_FOUND(N), {404, #{code => 'NOT_FOUND', message => ?TO_BIN([N, " NOT FOUND"])}}). --define(BAD_REQUEST(C, M), {400, #{code => C, message => ?TO_BIN(M)}}). +-define(SERVICE_UNAVAILABLE(C, M), {503, #{code => C, message => ?TO_BIN(M)}}). -define(TAGS, [<<"Trace">>]). namespace() -> "trace". @@ -148,8 +150,9 @@ schema("/trace/:name/download") -> #{schema => #{type => "string", format => "binary"}} } }, - 400 => emqx_dashboard_swagger:error_codes(['NODE_ERROR'], <<"Node Not Found">>), - 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>) + 404 => emqx_dashboard_swagger:error_codes( + ['NOT_FOUND', 'NODE_ERROR'], <<"Trace Name or Node Not Found">> + ) } } }; @@ -184,8 +187,15 @@ schema("/trace/:name/log") -> {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}, {meta, fields(bytes) ++ fields(position)} ], - 400 => emqx_dashboard_swagger:error_codes(['NODE_ERROR'], <<"Trace Log Failed">>), - 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>) + 400 => emqx_dashboard_swagger:error_codes( + ['BAD_REQUEST'], <<"Bad input parameter">> + ), + 404 => emqx_dashboard_swagger:error_codes( + ['NOT_FOUND', 'NODE_ERROR'], <<"Trace Name or Node Not Found">> + ), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], <<"Requested chunk size too big">> + ) } } }. @@ -313,12 +323,16 @@ fields(bytes) -> [ {bytes, hoconsc:mk( - integer(), + %% This seems to be the minimum max value we may encounter + %% across different OS + range(0, ?MAX_SINT32), #{ - desc => "Maximum number of bytes to store in request", + desc => "Maximum number of bytes to send in response", in => query, required => false, - default => 1000 + default => 1000, + minimum => 0, + maximum => ?MAX_SINT32 } )} ]; @@ -495,7 +509,7 @@ download_trace_log(get, #{bindings := #{name := Name}, query_string := Query}) - }, {200, Headers, {file_binary, ZipName, Binary}}; {error, not_found} -> - ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>) + ?NOT_FOUND(<<"Node">>) end; {error, not_found} -> ?NOT_FOUND(Name) @@ -579,11 +593,19 @@ stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) -> {200, #{meta => Meta, items => <<"">>}}; {error, not_found} -> ?NOT_FOUND(Name); + {error, enomem} -> + ?SLOG(warning, #{ + code => not_enough_mem, + msg => "Requested chunk size too big", + bytes => Bytes, + name => Name + }), + ?SERVICE_UNAVAILABLE('SERVICE_UNAVAILABLE', <<"Requested chunk size too big">>); {badrpc, nodedown} -> - ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>) + ?NOT_FOUND(<<"Node">>) end; {error, not_found} -> - ?BAD_REQUEST('NODE_ERROR', <<"Node not found">>) + ?NOT_FOUND(<<"Node">>) end. -spec get_trace_size() -> #{{node(), file:name_all()} => non_neg_integer()}. diff --git a/apps/emqx_management/src/emqx_mgmt_app.erl b/apps/emqx_management/src/emqx_mgmt_app.erl index 137f4502c..b4cf9091a 100644 --- a/apps/emqx_management/src/emqx_mgmt_app.erl +++ b/apps/emqx_management/src/emqx_mgmt_app.erl @@ -31,9 +31,7 @@ start(_Type, _Args) -> ok = mria_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity), case emqx_mgmt_auth:init_bootstrap_file() of ok -> - {ok, Sup} = emqx_mgmt_sup:start_link(), - ok = emqx_mgmt_cli:load(), - {ok, Sup}; + emqx_mgmt_sup:start_link(); {error, Reason} -> {error, Reason} end. diff --git a/apps/emqx_management/src/emqx_mgmt_util.erl b/apps/emqx_management/src/emqx_mgmt_util.erl index c0d9e6036..b81b39b07 100644 --- a/apps/emqx_management/src/emqx_mgmt_util.erl +++ b/apps/emqx_management/src/emqx_mgmt_util.erl @@ -302,7 +302,7 @@ page_params() -> name => limit, in => query, description => <<"Page size">>, - schema => #{type => integer, default => emqx_mgmt:max_row_limit()} + schema => #{type => integer, default => emqx_mgmt:default_row_limit()} } ]. diff --git a/apps/emqx_management/test/emqx_mgmt_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_SUITE.erl new file mode 100644 index 000000000..4619905cb --- /dev/null +++ b/apps/emqx_management/test/emqx_mgmt_SUITE.erl @@ -0,0 +1,387 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_mgmt_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-export([ident/1]). + +-define(FORMATFUN, {?MODULE, ident}). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_management]), + Config. + +end_per_suite(_) -> + emqx_mgmt_api_test_util:end_suite([emqx_management, emqx_conf]). + +init_per_testcase(TestCase, Config) -> + meck:expect(mria_mnesia, running_nodes, 0, [node()]), + emqx_common_test_helpers:init_per_testcase(?MODULE, TestCase, Config). + +end_per_testcase(TestCase, Config) -> + meck:unload(mria_mnesia), + emqx_common_test_helpers:end_per_testcase(?MODULE, TestCase, Config). + +t_list_nodes(init, Config) -> + meck:expect( + mria_mnesia, + cluster_nodes, + fun + (running) -> [node()]; + (stopped) -> ['stopped@node'] + end + ), + Config; +t_list_nodes('end', _Config) -> + ok. + +t_list_nodes(_) -> + NodeInfos = emqx_mgmt:list_nodes(), + Node = node(), + ?assertMatch( + [ + {Node, #{node := Node, node_status := 'running'}}, + {'stopped@node', #{node := 'stopped@node', node_status := 'stopped'}} + ], + NodeInfos + ). + +t_lookup_node(init, Config) -> + meck:new(os, [passthrough, unstick, no_link]), + OsType = os:type(), + meck:expect(os, type, 0, {win32, winME}), + [{os_type, OsType} | Config]; +t_lookup_node('end', Config) -> + %% We need to restore the original behavior so that rebar3 doesn't crash. If + %% we'd `meck:unload(os)` or not set `no_link` then `ct` crashes calling + %% `os` with "The code server called the unloaded module `os'". + OsType = ?config(os_type, Config), + meck:expect(os, type, 0, OsType), + ok. + +t_lookup_node(_) -> + Node = node(), + ?assertMatch( + #{node := Node, node_status := 'running', memory_total := 0}, + emqx_mgmt:lookup_node(node()) + ), + ?assertMatch( + {error, _}, + emqx_mgmt:lookup_node('fake@nohost') + ), + ok. + +t_list_brokers(_) -> + Node = node(), + ?assertMatch( + [{Node, #{node := Node, node_status := running, uptime := _}}], + emqx_mgmt:list_brokers() + ). + +t_lookup_broker(_) -> + Node = node(), + ?assertMatch( + #{node := Node, node_status := running, uptime := _}, + emqx_mgmt:lookup_broker(Node) + ). + +t_get_metrics(_) -> + Metrics = emqx_mgmt:get_metrics(), + ?assert(maps:size(Metrics) > 0), + ?assertMatch( + Metrics, maps:from_list(emqx_mgmt:get_metrics(node())) + ). + +t_lookup_client(init, Config) -> + setup_clients(Config); +t_lookup_client('end', Config) -> + disconnect_clients(Config). + +t_lookup_client(_Config) -> + [{Chan, Info, Stats}] = emqx_mgmt:lookup_client({clientid, <<"client1">>}, ?FORMATFUN), + ?assertEqual( + [{Chan, Info, Stats}], + emqx_mgmt:lookup_client({username, <<"user1">>}, ?FORMATFUN) + ), + ?assertEqual([], emqx_mgmt:lookup_client({clientid, <<"notfound">>}, ?FORMATFUN)), + meck:expect(mria_mnesia, running_nodes, 0, [node(), 'fake@nonode']), + ?assertMatch( + [_ | {error, nodedown}], emqx_mgmt:lookup_client({clientid, <<"client1">>}, ?FORMATFUN) + ). + +t_kickout_client(init, Config) -> + process_flag(trap_exit, true), + setup_clients(Config); +t_kickout_client('end', _Config) -> + ok. + +t_kickout_client(Config) -> + [C | _] = ?config(clients, Config), + ok = emqx_mgmt:kickout_client(<<"client1">>), + receive + {'EXIT', C, Reason} -> + ?assertEqual({shutdown, tcp_closed}, Reason); + Foo -> + error({unexpected, Foo}) + after 1000 -> + error(timeout) + end, + ?assertEqual({error, not_found}, emqx_mgmt:kickout_client(<<"notfound">>)). + +t_list_authz_cache(init, Config) -> + setup_clients(Config); +t_list_authz_cache('end', Config) -> + disconnect_clients(Config). + +t_list_authz_cache(_) -> + ?assertNotMatch({error, _}, emqx_mgmt:list_authz_cache(<<"client1">>)), + ?assertMatch({error, not_found}, emqx_mgmt:list_authz_cache(<<"notfound">>)). + +t_list_client_subscriptions(init, Config) -> + setup_clients(Config); +t_list_client_subscriptions('end', Config) -> + disconnect_clients(Config). + +t_list_client_subscriptions(Config) -> + [Client | _] = ?config(clients, Config), + ?assertEqual([], emqx_mgmt:list_client_subscriptions(<<"client1">>)), + emqtt:subscribe(Client, <<"t/#">>), + ?assertMatch({_, [{<<"t/#">>, _Opts}]}, emqx_mgmt:list_client_subscriptions(<<"client1">>)), + ?assertEqual({error, not_found}, emqx_mgmt:list_client_subscriptions(<<"notfound">>)). + +t_clean_cache(init, Config) -> + setup_clients(Config); +t_clean_cache('end', Config) -> + disconnect_clients(Config). + +t_clean_cache(_Config) -> + ?assertNotMatch( + {error, _}, + emqx_mgmt:clean_authz_cache(<<"client1">>) + ), + ?assertNotMatch( + {error, _}, + emqx_mgmt:clean_authz_cache_all() + ), + ?assertNotMatch( + {error, _}, + emqx_mgmt:clean_pem_cache_all() + ), + meck:expect(mria_mnesia, running_nodes, 0, [node(), 'fake@nonode']), + ?assertMatch( + {error, [{'fake@nonode', {error, _}}]}, + emqx_mgmt:clean_authz_cache_all() + ), + ?assertMatch( + {error, [{'fake@nonode', {error, _}}]}, + emqx_mgmt:clean_pem_cache_all() + ). + +t_set_client_props(init, Config) -> + setup_clients(Config); +t_set_client_props('end', Config) -> + disconnect_clients(Config). + +t_set_client_props(_Config) -> + ?assertEqual( + % [FIXME] not implemented at this point? + ignored, + emqx_mgmt:set_ratelimit_policy(<<"client1">>, foo) + ), + ?assertEqual( + {error, not_found}, + emqx_mgmt:set_ratelimit_policy(<<"notfound">>, foo) + ), + ?assertEqual( + % [FIXME] not implemented at this point? + ignored, + emqx_mgmt:set_quota_policy(<<"client1">>, foo) + ), + ?assertEqual( + {error, not_found}, + emqx_mgmt:set_quota_policy(<<"notfound">>, foo) + ), + ?assertEqual( + ok, + emqx_mgmt:set_keepalive(<<"client1">>, 3600) + ), + ?assertMatch( + {error, _}, + emqx_mgmt:set_keepalive(<<"client1">>, true) + ), + ?assertEqual( + {error, not_found}, + emqx_mgmt:set_keepalive(<<"notfound">>, 3600) + ), + ok. + +t_list_subscriptions_via_topic(init, Config) -> + setup_clients(Config); +t_list_subscriptions_via_topic('end', Config) -> + disconnect_clients(Config). + +t_list_subscriptions_via_topic(Config) -> + [Client | _] = ?config(clients, Config), + ?assertEqual([], emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)), + emqtt:subscribe(Client, <<"t/#">>), + ?assertMatch( + [{{<<"t/#">>, _SubPid}, _Opts}], + emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN) + ). + +t_pubsub_api(init, Config) -> + setup_clients(Config); +t_pubsub_api('end', Config) -> + disconnect_clients(Config). + +-define(TT(Topic), {Topic, #{qos => 0}}). + +t_pubsub_api(Config) -> + [Client | _] = ?config(clients, Config), + ?assertEqual([], emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN)), + ?assertMatch( + {subscribe, _, _}, + emqx_mgmt:subscribe(<<"client1">>, [?TT(<<"t/#">>), ?TT(<<"t1/#">>), ?TT(<<"t2/#">>)]) + ), + timer:sleep(100), + ?assertMatch( + [{{<<"t/#">>, _SubPid}, _Opts}], + emqx_mgmt:list_subscriptions_via_topic(<<"t/#">>, ?FORMATFUN) + ), + Message = emqx_message:make(?MODULE, 0, <<"t/foo">>, <<"helloworld">>, #{}, #{}), + emqx_mgmt:publish(Message), + Recv = + receive + {publish, #{client_pid := Client, payload := <<"helloworld">>}} -> + ok + after 100 -> + timeout + end, + ?assertEqual(ok, Recv), + ?assertEqual({error, channel_not_found}, emqx_mgmt:subscribe(<<"notfound">>, [?TT(<<"t/#">>)])), + ?assertNotMatch({error, _}, emqx_mgmt:unsubscribe(<<"client1">>, <<"t/#">>)), + ?assertEqual({error, channel_not_found}, emqx_mgmt:unsubscribe(<<"notfound">>, <<"t/#">>)), + Node = node(), + ?assertMatch( + {Node, [{<<"t1/#">>, _}, {<<"t2/#">>, _}]}, + emqx_mgmt:list_client_subscriptions(<<"client1">>) + ), + ?assertMatch( + {unsubscribe, [{<<"t1/#">>, _}, {<<"t2/#">>, _}]}, + emqx_mgmt:unsubscribe_batch(<<"client1">>, [<<"t1/#">>, <<"t2/#">>]) + ), + timer:sleep(100), + ?assertMatch([], emqx_mgmt:list_client_subscriptions(<<"client1">>)), + ?assertEqual( + {error, channel_not_found}, + emqx_mgmt:unsubscribe_batch(<<"notfound">>, [<<"t1/#">>, <<"t2/#">>]) + ). + +t_alarms(init, Config) -> + [ + emqx_mgmt:deactivate(Node, Name) + || {Node, ActiveAlarms} <- emqx_mgmt:get_alarms(activated), #{name := Name} <- ActiveAlarms + ], + emqx_mgmt:delete_all_deactivated_alarms(), + Config; +t_alarms('end', Config) -> + Config. + +t_alarms(_) -> + Node = node(), + ?assertEqual( + [{node(), []}], + emqx_mgmt:get_alarms(all) + ), + emqx_alarm:activate(foo), + ?assertMatch( + [{Node, [#{name := foo, activated := true, duration := _}]}], + emqx_mgmt:get_alarms(all) + ), + emqx_alarm:activate(bar), + ?assertMatch( + [{Node, [#{name := foo, activated := true}, #{name := bar, activated := true}]}], + sort_alarms(emqx_mgmt:get_alarms(all)) + ), + ?assertEqual( + ok, + emqx_mgmt:deactivate(node(), bar) + ), + ?assertMatch( + [{Node, [#{name := foo, activated := true}, #{name := bar, activated := false}]}], + sort_alarms(emqx_mgmt:get_alarms(all)) + ), + ?assertMatch( + [{Node, [#{name := foo, activated := true}]}], + emqx_mgmt:get_alarms(activated) + ), + ?assertMatch( + [{Node, [#{name := bar, activated := false}]}], + emqx_mgmt:get_alarms(deactivated) + ), + ?assertEqual( + [ok], + emqx_mgmt:delete_all_deactivated_alarms() + ), + ?assertMatch( + [{Node, [#{name := foo, activated := true}]}], + emqx_mgmt:get_alarms(all) + ), + ?assertEqual( + {error, not_found}, + emqx_mgmt:deactivate(node(), bar) + ). + +t_banned(_) -> + Banned = #{ + who => {clientid, <<"TestClient">>}, + by => <<"banned suite">>, + reason => <<"test">>, + at => erlang:system_time(second), + until => erlang:system_time(second) + 1 + }, + ?assertMatch( + {ok, _}, + emqx_mgmt:create_banned(Banned) + ), + ?assertEqual( + ok, + emqx_mgmt:delete_banned({clientid, <<"TestClient">>}) + ). + +%%% helpers +ident(Arg) -> + Arg. + +sort_alarms([{Node, Alarms}]) -> + [{Node, lists:sort(fun(#{activate_at := A}, #{activate_at := B}) -> A < B end, Alarms)}]. + +setup_clients(Config) -> + {ok, C} = emqtt:start_link([{clientid, <<"client1">>}, {username, <<"user1">>}]), + {ok, _} = emqtt:connect(C), + [{clients, [C]} | Config]. + +disconnect_clients(Config) -> + Clients = ?config(clients, Config), + lists:foreach(fun emqtt:disconnect/1, Clients). diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index a8bbfa6d9..4d0262e6a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -67,7 +67,7 @@ t_cluster_query(_Config) -> %% assert: AllPage = Page1 + Page2 + Page3 + Page4 %% !!!Note: this equation requires that the queried tables must be ordered_set - {200, ClientsPage2} = query_clients(Node1, #{<<"page">> => 2, <<"limit">> => 5}), + {200, ClientsPage2} = query_clients(Node1, #{<<"page">> => <<"2">>, <<"limit">> => 5}), {200, ClientsPage3} = query_clients(Node2, #{<<"page">> => 3, <<"limit">> => 5}), {200, ClientsPage4} = query_clients(Node1, #{<<"page">> => 4, <<"limit">> => 5}), GetClientIds = fun(L) -> lists:map(fun(#{clientid := Id}) -> Id end, L) end, @@ -79,6 +79,78 @@ t_cluster_query(_Config) -> ) ), + %% Scroll past count + {200, ClientsPage10} = query_clients(Node1, #{<<"page">> => <<"10">>, <<"limit">> => 5}), + ?assertEqual( + #{data => [], meta => #{page => 10, limit => 5, count => 20, hasnext => false}}, + ClientsPage10 + ), + + %% Node queries + {200, ClientsNode2} = query_clients(Node1, #{<<"node">> => Node2}), + ?assertEqual({200, ClientsNode2}, query_clients(Node2, #{<<"node">> => Node2})), + ?assertMatch( + #{page := 1, limit := 100, count := 10}, + maps:get(meta, ClientsNode2) + ), + ?assertMatch(10, length(maps:get(data, ClientsNode2))), + + {200, ClientsNode2Page1} = query_clients(Node2, #{<<"node">> => Node2, <<"limit">> => 5}), + {200, ClientsNode2Page2} = query_clients(Node1, #{ + <<"node">> => Node2, <<"page">> => <<"2">>, <<"limit">> => 5 + }), + {200, ClientsNode2Page3} = query_clients(Node2, #{ + <<"node">> => Node2, <<"page">> => 3, <<"limit">> => 5 + }), + {200, ClientsNode2Page4} = query_clients(Node1, #{ + <<"node">> => Node2, <<"page">> => 4, <<"limit">> => 5 + }), + ?assertEqual( + GetClientIds(maps:get(data, ClientsNode2)), + GetClientIds( + lists:append([ + maps:get(data, Page) + || Page <- [ + ClientsNode2Page1, + ClientsNode2Page2, + ClientsNode2Page3, + ClientsNode2Page4 + ] + ]) + ) + ), + + %% Scroll past count + {200, ClientsNode2Page10} = query_clients(Node1, #{ + <<"node">> => Node2, <<"page">> => <<"10">>, <<"limit">> => 5 + }), + ?assertEqual( + #{data => [], meta => #{page => 10, limit => 5, count => 10, hasnext => false}}, + ClientsNode2Page10 + ), + + %% Query with bad params + ?assertEqual( + {400, #{ + code => <<"INVALID_PARAMETER">>, + message => <<"page_limit_invalid">> + }}, + query_clients(Node1, #{<<"page">> => -1}) + ), + ?assertEqual( + {400, #{ + code => <<"INVALID_PARAMETER">>, + message => <<"page_limit_invalid">> + }}, + query_clients(Node1, #{<<"node">> => Node1, <<"page">> => -1}) + ), + + %% Query bad node + ?assertMatch( + {500, #{code := <<"NODE_DOWN">>}}, + query_clients(Node1, #{<<"node">> => 'nonode@nohost'}) + ), + %% exact match can return non-zero total {200, ClientsNode1} = query_clients(Node2, #{<<"username">> => <<"corenode1@127.0.0.1">>}), ?assertMatch( @@ -87,11 +159,11 @@ t_cluster_query(_Config) -> ), %% fuzzy searching can't return total - {200, ClientsNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}), - MetaNode2 = maps:get(meta, ClientsNode2), + {200, ClientsFuzzyNode2} = query_clients(Node2, #{<<"like_username">> => <<"corenode2">>}), + MetaNode2 = maps:get(meta, ClientsFuzzyNode2), ?assertNotMatch(#{count := _}, MetaNode2), ?assertMatch(#{hasnext := false}, MetaNode2), - ?assertMatch(10, length(maps:get(data, ClientsNode2))), + ?assertMatch(10, length(maps:get(data, ClientsFuzzyNode2))), _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1), _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs2) @@ -101,6 +173,23 @@ t_cluster_query(_Config) -> end, ok. +t_bad_rpc(_) -> + emqx_mgmt_api_test_util:init_suite(), + process_flag(trap_exit, true), + ClientLs1 = [start_emqtt_client(node(), I, 1883) || I <- lists:seq(1, 10)], + Path = emqx_mgmt_api_test_util:api_path(["clients?limit=2&page=2"]), + try + meck:expect(mria_mnesia, running_nodes, 0, ['fake@nohost']), + {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path), + %% good cop, bad cop + meck:expect(mria_mnesia, running_nodes, 0, [node(), 'fake@nohost']), + {error, {_, 500, _}} = emqx_mgmt_api_test_util:request_api(get, Path) + after + _ = lists:foreach(fun(C) -> emqtt:disconnect(C) end, ClientLs1), + meck:unload(mria_mnesia), + emqx_mgmt_api_test_util:end_suite() + end. + %%-------------------------------------------------------------------- %% helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl index 2c61651bf..69ace16e8 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl @@ -62,5 +62,5 @@ get_alarms(AssertCount, Activated) -> Limit = maps:get(<<"limit">>, Meta), Count = maps:get(<<"count">>, Meta), ?assertEqual(Page, 1), - ?assertEqual(Limit, emqx_mgmt:max_row_limit()), + ?assertEqual(Limit, emqx_mgmt:default_row_limit()), ?assert(Count >= AssertCount). diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 16ba99ad6..9f26f8542 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -64,7 +64,7 @@ t_clients(_) -> ClientsLimit = maps:get(<<"limit">>, ClientsMeta), ClientsCount = maps:get(<<"count">>, ClientsMeta), ?assertEqual(ClientsPage, 1), - ?assertEqual(ClientsLimit, emqx_mgmt:max_row_limit()), + ?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()), ?assertEqual(ClientsCount, 2), %% get /clients/:clientid @@ -78,7 +78,14 @@ t_clients(_) -> %% delete /clients/:clientid kickout Client2Path = emqx_mgmt_api_test_util:api_path(["clients", binary_to_list(ClientId2)]), {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client2Path), - timer:sleep(300), + Kick = + receive + {'EXIT', C2, _} -> + ok + after 300 -> + timeout + end, + ?assertEqual(ok, Kick), AfterKickoutResponse2 = emqx_mgmt_api_test_util:request_api(get, Client2Path), ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse2), @@ -107,7 +114,7 @@ t_clients(_) -> SubscribeBody ), timer:sleep(100), - [{AfterSubTopic, #{qos := AfterSubQos}}] = emqx_mgmt:lookup_subscriptions(ClientId1), + {_, [{AfterSubTopic, #{qos := AfterSubQos}}]} = emqx_mgmt:list_client_subscriptions(ClientId1), ?assertEqual(AfterSubTopic, Topic), ?assertEqual(AfterSubQos, Qos), @@ -152,7 +159,7 @@ t_clients(_) -> UnSubscribeBody ), timer:sleep(100), - ?assertEqual([], emqx_mgmt:lookup_subscriptions(Client1)), + ?assertEqual([], emqx_mgmt:list_client_subscriptions(ClientId1)), %% testcase cleanup, kickout client1 {ok, _} = emqx_mgmt_api_test_util:request_api(delete, Client1Path), @@ -247,6 +254,49 @@ t_keepalive(_Config) -> emqtt:disconnect(C1), ok. +t_client_id_not_found(_Config) -> + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Http = {"HTTP/1.1", 404, "Not Found"}, + Body = "{\"code\":\"CLIENTID_NOT_FOUND\",\"message\":\"Client ID not found\"}", + + PathFun = fun(Suffix) -> + emqx_mgmt_api_test_util:api_path(["clients", "no_existed_clientid"] ++ Suffix) + end, + ReqFun = fun(Method, Path) -> + emqx_mgmt_api_test_util:request_api( + Method, Path, "", AuthHeader, [], #{return_all => true} + ) + end, + + PostFun = fun(Method, Path, Data) -> + emqx_mgmt_api_test_util:request_api( + Method, Path, "", AuthHeader, Data, #{return_all => true} + ) + end, + + %% Client lookup + ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun([]))), + %% Client kickout + ?assertMatch({error, {Http, _, Body}}, ReqFun(delete, PathFun([]))), + %% Client Subscription list + ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun(["subscriptions"]))), + %% AuthZ Cache lookup + ?assertMatch({error, {Http, _, Body}}, ReqFun(get, PathFun(["authorization", "cache"]))), + %% AuthZ Cache clean + ?assertMatch({error, {Http, _, Body}}, ReqFun(delete, PathFun(["authorization", "cache"]))), + %% Client Subscribe + SubBody = #{topic => <<"testtopic">>, qos => 1, nl => 1, rh => 1}, + ?assertMatch({error, {Http, _, Body}}, PostFun(post, PathFun(["subscribe"]), SubBody)), + ?assertMatch( + {error, {Http, _, Body}}, PostFun(post, PathFun(["subscribe", "bulk"]), [SubBody]) + ), + %% Client Unsubscribe + UnsubBody = #{topic => <<"testtopic">>}, + ?assertMatch({error, {Http, _, Body}}, PostFun(post, PathFun(["unsubscribe"]), UnsubBody)), + ?assertMatch( + {error, {Http, _, Body}}, PostFun(post, PathFun(["unsubscribe", "bulk"]), [UnsubBody]) + ). + time_string_to_epoch_millisecond(DateTime) -> time_string_to_epoch(DateTime, millisecond). diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index 2ab213e30..ccfa30037 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -57,7 +57,7 @@ t_subscription_api(Config) -> Data = emqx_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, Data), ?assertEqual(1, maps:get(<<"page">>, Meta)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta)), + ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), ?assertEqual(2, maps:get(<<"count">>, Meta)), Subscriptions = maps:get(<<"data">>, Data), ?assertEqual(length(Subscriptions), 2), @@ -95,7 +95,7 @@ t_subscription_api(Config) -> DataTopic2 = #{<<"meta">> := Meta2} = request_json(get, QS, Headers), ?assertEqual(1, maps:get(<<"page">>, Meta2)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta2)), + ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta2)), ?assertEqual(1, maps:get(<<"count">>, Meta2)), SubscriptionsList2 = maps:get(<<"data">>, DataTopic2), ?assertEqual(length(SubscriptionsList2), 1). @@ -120,7 +120,7 @@ t_subscription_fuzzy_search(Config) -> MatchData1 = #{<<"meta">> := MatchMeta1} = request_json(get, MatchQs, Headers), ?assertEqual(1, maps:get(<<"page">>, MatchMeta1)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, MatchMeta1)), + ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, MatchMeta1)), %% count is undefined in fuzzy searching ?assertNot(maps:is_key(<<"count">>, MatchMeta1)), ?assertMatch(3, length(maps:get(<<"data">>, MatchData1))), diff --git a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl index 8f9b224ef..0c2e684b4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl @@ -52,7 +52,7 @@ t_nodes_api(Config) -> RoutesData = emqx_json:decode(Response, [return_maps]), Meta = maps:get(<<"meta">>, RoutesData), ?assertEqual(1, maps:get(<<"page">>, Meta)), - ?assertEqual(emqx_mgmt:max_row_limit(), maps:get(<<"limit">>, Meta)), + ?assertEqual(emqx_mgmt:default_row_limit(), maps:get(<<"limit">>, Meta)), ?assertEqual(1, maps:get(<<"count">>, Meta)), Data = maps:get(<<"data">>, RoutesData), Route = erlang:hd(Data), diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index 6962a9043..162d07aaa 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -19,9 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). -include_lib("kernel/include/file.hrl"). -include_lib("stdlib/include/zip.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -225,12 +223,12 @@ t_log_file(_Config) -> ]}, zip:table(Binary2) ), - {error, {_, 400, _}} = + {error, {_, 404, _}} = request_api( get, - api_path("trace/test_client_id/download?node=unknonwn_node") + api_path("trace/test_client_id/download?node=unknown_node") ), - {error, {_, 400, _}} = + {error, {_, 404, _}} = request_api( get, % known atom but unknown node @@ -296,12 +294,21 @@ t_stream_log(_Config) -> #{<<"meta">> := Meta1, <<"items">> := Bin1} = json(Binary1), ?assertEqual(#{<<"position">> => 30, <<"bytes">> => 10}, Meta1), ?assertEqual(10, byte_size(Bin1)), - {error, {_, 400, _}} = + ct:pal("~p vs ~p", [Bin, Bin1]), + %% in theory they could be the same but we know they shouldn't + ?assertNotEqual(Bin, Bin1), + BadReqPath = api_path("trace/test_stream_log/log?&bytes=1000000000000"), + {error, {_, 400, _}} = request_api(get, BadReqPath), + meck:new(file, [passthrough, unstick]), + meck:expect(file, read, 2, {error, enomem}), + {error, {_, 503, _}} = request_api(get, Path), + meck:unload(file), + {error, {_, 404, _}} = request_api( get, - api_path("trace/test_stream_log/log?node=unknonwn_node") + api_path("trace/test_stream_log/log?node=unknown_node") ), - {error, {_, 400, _}} = + {error, {_, 404, _}} = request_api( get, % known atom but not a node diff --git a/apps/emqx_modules/src/emqx_modules.app.src b/apps/emqx_modules/src/emqx_modules.app.src index 20f8a76fc..60d36d673 100644 --- a/apps/emqx_modules/src/emqx_modules.app.src +++ b/apps/emqx_modules/src/emqx_modules.app.src @@ -1,9 +1,9 @@ %% -*- mode: erlang -*- {application, emqx_modules, [ {description, "EMQX Modules"}, - {vsn, "5.0.9"}, + {vsn, "5.0.10"}, {modules, []}, - {applications, [kernel, stdlib, emqx]}, + {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_modules_app, []}}, {registered, [emqx_modules_sup]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src index 3120b8503..7acf7433b 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugin_libs, [ {description, "EMQX Plugin utility libs"}, - {vsn, "4.3.5"}, + {vsn, "4.3.6"}, {modules, []}, {applications, [kernel, stdlib]}, {env, []} diff --git a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl index 969374309..a60c94a7b 100644 --- a/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl +++ b/apps/emqx_plugin_libs/src/emqx_plugin_libs_rule.erl @@ -31,7 +31,8 @@ proc_sql_param_str/2, proc_cql_param_str/2, split_insert_sql/1, - detect_sql_type/1 + detect_sql_type/1, + proc_batch_sql/3 ]). %% type converting @@ -164,6 +165,20 @@ detect_sql_type(SQL) -> {error, invalid_sql} end. +-spec proc_batch_sql( + BatchReqs :: list({atom(), map()}), + InsertPart :: binary(), + Tokens :: tmpl_token() +) -> InsertSQL :: binary(). +proc_batch_sql(BatchReqs, InsertPart, Tokens) -> + ValuesPart = erlang:iolist_to_binary( + lists:join(", ", [ + emqx_plugin_libs_rule:proc_sql_param_str(Tokens, Msg) + || {_, Msg} <- BatchReqs + ]) + ), + <>. + unsafe_atom_key(Key) when is_atom(Key) -> Key; unsafe_atom_key(Key) when is_binary(Key) -> diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index de56099ba..ed893c80d 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl index 8b3cca8fd..9d9d045de 100644 --- a/apps/emqx_plugins/src/emqx_plugins_schema.erl +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -78,11 +78,11 @@ states(_) -> undefined. install_dir(type) -> string(); install_dir(required) -> false; %% runner's root dir -install_dir(default) -> "plugins"; +install_dir(default) -> <<"plugins">>; install_dir(T) when T =/= desc -> undefined; install_dir(desc) -> ?DESC(install_dir). check_interval(type) -> emqx_schema:duration(); -check_interval(default) -> "5s"; +check_interval(default) -> <<"5s">>; check_interval(T) when T =/= desc -> undefined; check_interval(desc) -> ?DESC(check_interval). diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 013de63fb..6970ba777 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,10 +2,10 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, []}, {registered, [emqx_prometheus_sup]}, - {applications, [kernel, stdlib, prometheus, emqx]}, + {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, {mod, {emqx_prometheus_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus.erl b/apps/emqx_prometheus/src/emqx_prometheus.erl index 4712a43c8..62e6f1d9a 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus.erl @@ -590,20 +590,7 @@ emqx_vm() -> ]. emqx_vm_data() -> - Idle = - case cpu_sup:util([detailed]) of - %% Not support for Windows - {_, 0, 0, _} -> 0; - {_Num, _Use, IdleList, _} -> ?C(idle, IdleList) - end, - RunQueue = erlang:statistics(run_queue), - [ - {run_queue, RunQueue}, - %% XXX: Plan removed at v5.0 - {process_total_messages, 0}, - {cpu_idle, Idle}, - {cpu_use, 100 - Idle} - ] ++ emqx_vm:mem_info(). + emqx_mgmt:vm_stats(). emqx_cluster() -> [ diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index fcda5dea0..6ced0bf42 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -40,7 +40,7 @@ fields("prometheus") -> ?HOCON( string(), #{ - default => "http://127.0.0.1:9091", + default => <<"http://127.0.0.1:9091">>, required => true, validator => fun ?MODULE:validate_push_gateway_server/1, desc => ?DESC(push_gateway_server) @@ -50,7 +50,7 @@ fields("prometheus") -> ?HOCON( emqx_schema:duration_ms(), #{ - default => "15s", + default => <<"15s">>, required => true, desc => ?DESC(interval) } diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 4618e94a6..cb26c7f09 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index b25725c41..bb4eee57d 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -468,7 +468,10 @@ flush(Data0) -> queue := Q0 } = Data0, Data1 = cancel_flush_timer(Data0), - case {queue_count(Q0), is_inflight_full(InflightTID)} of + CurrentCount = queue_count(Q0), + IsFull = is_inflight_full(InflightTID), + ?tp(buffer_worker_flush, #{queue_count => CurrentCount, is_full => IsFull}), + case {CurrentCount, IsFull} of {0, _} -> {keep_state, Data1}; {_, true} -> @@ -595,8 +598,12 @@ do_flush( result => Result } ), - case queue_count(Q1) > 0 of + CurrentCount = queue_count(Q1), + case CurrentCount > 0 of true -> + ?tp(buffer_worker_flush_ack_reflush, #{ + batch_or_query => Request, result => Result, queue_count => CurrentCount + }), flush_worker(self()); false -> ok @@ -666,19 +673,26 @@ do_flush(Data0, #{ {Data1, WorkerMRef} = ensure_async_worker_monitored(Data0, Result), store_async_worker_reference(InflightTID, Ref, WorkerMRef), emqx_resource_metrics:queuing_set(Id, Index, queue_count(Q1)), + CurrentCount = queue_count(Q1), ?tp( buffer_worker_flush_ack, #{ batch_or_query => Batch, - result => Result + result => Result, + queue_count => CurrentCount } ), - CurrentCount = queue_count(Q1), Data2 = case {CurrentCount > 0, CurrentCount >= BatchSize} of {false, _} -> Data1; {true, true} -> + ?tp(buffer_worker_flush_ack_reflush, #{ + batch_or_query => Batch, + result => Result, + queue_count => CurrentCount, + batch_size => BatchSize + }), flush_worker(self()), Data1; {true, false} -> diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index a042bfb67..984b3b04a 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -2291,6 +2291,67 @@ t_expiration_retry_batch_multiple_times(_Config) -> ), ok. +t_recursive_flush(_Config) -> + emqx_connector_demo:set_callback_mode(async_if_possible), + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + query_mode => async, + batch_size => 1, + batch_time => 10_000, + worker_pool_size => 1 + } + ), + do_t_recursive_flush(). + +t_recursive_flush_batch(_Config) -> + emqx_connector_demo:set_callback_mode(async_if_possible), + {ok, _} = emqx_resource:create( + ?ID, + ?DEFAULT_RESOURCE_GROUP, + ?TEST_RESOURCE, + #{name => test_resource}, + #{ + query_mode => async, + batch_size => 2, + batch_time => 10_000, + worker_pool_size => 1 + } + ), + do_t_recursive_flush(). + +do_t_recursive_flush() -> + ?check_trace( + begin + Timeout = 1_000, + Pid = spawn_link(fun S() -> + emqx_resource:query(?ID, {inc_counter, 1}), + S() + end), + %% we want two reflushes to happen before we analyze the + %% trace, so that we get a single full interaction + {ok, _} = snabbkaffe:block_until( + ?match_n_events(2, #{?snk_kind := buffer_worker_flush_ack_reflush}), Timeout + ), + unlink(Pid), + exit(Pid, kill), + ok + end, + fun(Trace) -> + %% check that a recursive flush leads to a new call to flush/1 + Pairs = ?find_pairs( + #{?snk_kind := buffer_worker_flush_ack_reflush}, + #{?snk_kind := buffer_worker_flush}, + Trace + ), + ?assert(lists:any(fun(E) -> E end, [true || {pair, _, _} <- Pairs])) + end + ), + ok. + %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index d151ad4e7..8bdae6d7f 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,10 +2,10 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.9"}, + {vsn, "5.0.10"}, {modules, []}, {registered, [emqx_retainer_sup]}, - {applications, [kernel, stdlib, emqx]}, + {applications, [kernel, stdlib, emqx, emqx_ctl]}, {mod, {emqx_retainer_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index fa11b00f4..7b1337140 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -166,7 +166,7 @@ config(put, #{body := Body}) -> %%------------------------------------------------------------------------------ lookup_retained(get, #{query_string := Qs}) -> Page = maps:get(<<"page">>, Qs, 1), - Limit = maps:get(<<"limit">>, Qs, emqx_mgmt:max_row_limit()), + Limit = maps:get(<<"limit">>, Qs, emqx_mgmt:default_row_limit()), {ok, Msgs} = emqx_retainer_mnesia:page_read(undefined, undefined, Page, Limit), {200, #{ data => [format_message(Msg) || Msg <- Msgs], diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index 472ecc284..dbe1ad9d5 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -41,13 +41,13 @@ fields("retainer") -> sc( emqx_schema:duration_ms(), msg_expiry_interval, - "0s" + <<"0s">> )}, {msg_clear_interval, sc( emqx_schema:duration_ms(), msg_clear_interval, - "0s" + <<"0s">> )}, {flow_control, sc( @@ -59,7 +59,7 @@ fields("retainer") -> sc( emqx_schema:bytesize(), max_payload_size, - "1MB" + <<"1MB">> )}, {stop_publish_clear_msg, sc( diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index ee1544223..06ed059a4 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,10 +2,10 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, - {applications, [kernel, stdlib, rulesql, getopt]}, + {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]}, {mod, {emqx_rule_engine_app, []}}, {env, []}, {licenses, ["Apache-2.0"]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl index d6913cbc6..2281eea53 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_schema.erl @@ -51,7 +51,7 @@ fields("rule_engine") -> ?HOCON( emqx_schema:duration_ms(), #{ - default => "10s", + default => <<"10s">>, desc => ?DESC("rule_engine_jq_function_default_timeout") } )}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src index 866655b61..170a4bb02 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src @@ -1,7 +1,7 @@ {application, emqx_slow_subs, [ {description, "EMQX Slow Subscribers Statistics"}, % strict semver, bump manually! - {vsn, "1.0.2"}, + {vsn, "1.0.3"}, {modules, []}, {registered, [emqx_slow_subs_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index 8ae015ae4..9e9e6488a 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -30,13 +30,13 @@ fields("slow_subs") -> {threshold, sc( emqx_schema:duration_ms(), - "500ms", + <<"500ms">>, threshold )}, {expire_interval, sc( emqx_schema:duration_ms(), - "300s", + <<"300s">>, expire_interval )}, {top_k_num, diff --git a/apps/emqx_statsd/src/emqx_statsd.app.src b/apps/emqx_statsd/src/emqx_statsd.app.src index 638c5a33b..27f842ce2 100644 --- a/apps/emqx_statsd/src/emqx_statsd.app.src +++ b/apps/emqx_statsd/src/emqx_statsd.app.src @@ -1,14 +1,15 @@ %% -*- mode: erlang -*- {application, emqx_statsd, [ {description, "EMQX Statsd"}, - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {registered, []}, {mod, {emqx_statsd_app, []}}, {applications, [ kernel, stdlib, estatsd, - emqx + emqx, + emqx_management ]}, {env, []}, {modules, []}, diff --git a/apps/emqx_statsd/src/emqx_statsd.erl b/apps/emqx_statsd/src/emqx_statsd.erl index defaf78e0..770320ddd 100644 --- a/apps/emqx_statsd/src/emqx_statsd.erl +++ b/apps/emqx_statsd/src/emqx_statsd.erl @@ -105,7 +105,7 @@ handle_info( timer := Ref } ) -> - Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_vm_data(), + Metrics = emqx_metrics:all() ++ emqx_stats:getstats() ++ emqx_mgmt:vm_stats(), SampleRate = SampleTimeInterval / FlushTimeInterval, StatsdMetrics = [ {gauge, Name, Value, SampleRate, []} @@ -129,20 +129,6 @@ terminate(_Reason, #{estatsd_pid := Pid}) -> %% Internal function %%------------------------------------------------------------------------------ -emqx_vm_data() -> - Idle = - case cpu_sup:util([detailed]) of - %% Not support for Windows - {_, 0, 0, _} -> 0; - {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0) - end, - RunQueue = erlang:statistics(run_queue), - [ - {run_queue, RunQueue}, - {cpu_idle, Idle}, - {cpu_use, 100 - Idle} - ] ++ emqx_vm:mem_info(). - ensure_timer(State = #{sample_time_interval := SampleTimeInterval}) -> State#{timer => emqx_misc:start_timer(SampleTimeInterval, ?SAMPLE_TIMEOUT)}. diff --git a/apps/emqx_statsd/src/emqx_statsd_api.erl b/apps/emqx_statsd/src/emqx_statsd_api.erl index b1b3601aa..e65c93432 100644 --- a/apps/emqx_statsd/src/emqx_statsd_api.erl +++ b/apps/emqx_statsd/src/emqx_statsd_api.erl @@ -77,9 +77,9 @@ statsd_config_schema() -> statsd_example() -> #{ enable => true, - flush_time_interval => "30s", - sample_time_interval => "30s", - server => "127.0.0.1:8125", + flush_time_interval => <<"30s">>, + sample_time_interval => <<"30s">>, + server => <<"127.0.0.1:8125">>, tags => #{} }. diff --git a/apps/emqx_statsd/src/emqx_statsd_schema.erl b/apps/emqx_statsd/src/emqx_statsd_schema.erl index 1e5aa6e5f..e44f94954 100644 --- a/apps/emqx_statsd/src/emqx_statsd_schema.erl +++ b/apps/emqx_statsd/src/emqx_statsd_schema.erl @@ -61,12 +61,12 @@ server() -> emqx_schema:servers_sc(Meta, ?SERVER_PARSE_OPTS). sample_interval(type) -> emqx_schema:duration_ms(); -sample_interval(default) -> "30s"; +sample_interval(default) -> <<"30s">>; sample_interval(desc) -> ?DESC(?FUNCTION_NAME); sample_interval(_) -> undefined. flush_interval(type) -> emqx_schema:duration_ms(); -flush_interval(default) -> "30s"; +flush_interval(default) -> <<"30s">>; flush_interval(desc) -> ?DESC(?FUNCTION_NAME); flush_interval(_) -> undefined. diff --git a/bin/emqx b/bin/emqx index 0f6a18437..9211bd338 100755 --- a/bin/emqx +++ b/bin/emqx @@ -76,6 +76,12 @@ logwarn() { fi } +logdebug() { + if [ "$DEBUG" -eq 1 ]; then + echo "DEBUG: $*" + fi +} + die() { set +x logerr "$1" @@ -453,9 +459,38 @@ if [ "$IS_ENTERPRISE" = 'yes' ]; then CONF_KEYS+=( 'license.key' ) fi +## To be backward compatible, read and then unset EMQX_NODE_NAME +if [ -n "${EMQX_NODE_NAME:-}" ]; then + export EMQX_NODE__NAME="${EMQX_NODE_NAME}" + unset EMQX_NODE_NAME +fi + # Turn off debug as the ps output can be quite noisy set +x + +## Find the running node from 'ps -ef' +## * The grep args like '[e]mqx' but not 'emqx' is to avoid greping the grep command itself +## * The running 'remsh' and 'nodetool' processes must be excluded +if [ -n "${EMQX_NODE__NAME:-}" ]; then + # if node name is provided, filter by node name + # shellcheck disable=SC2009 + PS_LINE="$(ps -ef | $GREP '[e]mqx' | $GREP -v -E '(remsh|nodetool)' | $GREP -E "\s\-s?name\s${EMQX_NODE__NAME}" | $GREP -oE "\-[r]oot ${RUNNER_ROOT_DIR}.*" || true)" +else + # shellcheck disable=SC2009 + PS_LINE="$(ps -ef | $GREP '[e]mqx' | $GREP -v -E '(remsh|nodetool)' | $GREP -oE "\-[r]oot ${RUNNER_ROOT_DIR}.*" || true)" +fi +logdebug "PS_LINE=$PS_LINE" +RUNNING_NODES_COUNT="$(echo -e "$PS_LINE" | sed '/^\s*$/d' | wc -l)" +[ "$RUNNING_NODES_COUNT" -gt 1 ] && logdebug "More than one running node found: count=$RUNNING_NODES_COUNT" + if [ "$IS_BOOT_COMMAND" = 'yes' ]; then + if [ "$RUNNING_NODES_COUNT" -gt 0 ] && [ "$COMMAND" != 'check_config' ]; then + running_node_name=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' || true) + if [ -n "$running_node_name" ] && [ "$running_node_name" = "${EMQX_NODE__NAME:-}" ]; then + echo "Node ${running_node_name} is already running!" + exit 1 + fi + fi [ -f "$EMQX_ETC_DIR"/emqx.conf ] || die "emqx.conf is not found in $EMQX_ETC_DIR" 1 maybe_use_portable_dynlibs if [ "${EMQX_BOOT_CONFIGS:-}" = '' ]; then @@ -486,14 +521,7 @@ else # then update the config in the file to 'node.name = "emqx@local.net"', after this change, # there would be no way stop the running node 'emqx@127.0.0.1', because 'emqx stop' command # would try to stop the new node instead. - # * The primary grep pattern is $RUNNER_ROOT_DIR because one can start multiple nodes at the same time - # * The grep args like '[e]mqx' but not 'emqx' is to avoid greping the grep command itself - # * The running 'remsh' and 'nodetool' processes must be excluded - # shellcheck disable=SC2009 - PS_LINE="$(ps -ef | $GREP '[e]mqx' | $GREP -v -E '(remsh|nodetool)' | $GREP -oE "\-[r]oot ${RUNNER_ROOT_DIR}.*" || true)" - [ "$DEBUG" -eq 1 ] && echo "EMQX processes: $PS_LINE" - running_nodes_count="$(echo -e "$PS_LINE" | wc -l)" - if [ "$running_nodes_count" -eq 1 ]; then + if [ "$RUNNING_NODES_COUNT" -eq 1 ]; then ## only one emqx node is running, get running args from 'ps -ef' output tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' || true) tmp_cookie=$(echo -e "$PS_LINE" | $GREP -oE "\s\-setcookie.*" | awk '{print $2}' || true) @@ -509,14 +537,22 @@ else ## Make the format like what call_hocon multi_get prints out, but only need 4 args EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}" else - ## None or more than one node is running, resolve from boot config - ## we have no choiece but to read the bootstrap config (with environment overrides available in the current shell) + if [ "$RUNNING_NODES_COUNT" -gt 1 ]; then + if [ -z "${EMQX_NODE__NAME:-}" ]; then + tmp_nodenames=$(echo -e "$PS_LINE" | $GREP -oE "\s\-s?name.*" | awk '{print $2}' | tr '\n' ' ') + logerr "More than one EMQX node found running (root dir: ${RUNNER_ROOT_DIR})" + logerr "Running nodes: $tmp_nodenames" + logerr "Make sure environment variable EMQX_NODE__NAME is set to indicate for which node this command is intended." + exit 1 + fi + fi + ## We have no choiece but to read the bootstrap config (with environment overrides available in the current shell) [ -f "$EMQX_ETC_DIR"/emqx.conf ] || die "emqx.conf is not found in $EMQX_ETC_DIR" 1 maybe_use_portable_dynlibs EMQX_BOOT_CONFIGS="$(call_hocon -s "$SCHEMA_MOD" -c "$EMQX_ETC_DIR"/emqx.conf multi_get "${CONF_KEYS[@]}")" fi fi -[ "$DEBUG" -eq 1 ] && echo "EMQX_BOOT_CONFIGS: $EMQX_BOOT_CONFIGS" +logdebug "EMQX_BOOT_CONFIGS: $EMQX_BOOT_CONFIGS" [ "$DEBUG" -eq 1 ] && set -x get_boot_config() { @@ -866,11 +902,6 @@ maybe_log_to_console() { fi } -## To be backward compatible, read and then unset EMQX_NODE_NAME -if [ -n "${EMQX_NODE_NAME:-}" ]; then - export EMQX_NODE__NAME="${EMQX_NODE_NAME}" - unset EMQX_NODE_NAME -fi ## Possible ways to configure emqx node name: ## 1. configure node.name in emqx.conf ## 2. override with environment variable EMQX_NODE__NAME @@ -913,11 +944,14 @@ if [ -z "$COOKIE" ]; then COOKIE="$(get_boot_config 'node.cookie')" fi [ -z "$COOKIE" ] && COOKIE="$EMQX_DEFAULT_ERLANG_COOKIE" -if [ $IS_BOOT_COMMAND = 'yes' ] && [ "$COOKIE" = "$EMQX_DEFAULT_ERLANG_COOKIE" ]; then - logwarn "Default (insecure) Erlang cookie is in use." - logwarn "Configure node.cookie in $EMQX_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE" - logwarn "NOTE: Use the same cookie for all nodes in the cluster." -fi + +maybe_warn_default_cookie() { + if [ $IS_BOOT_COMMAND = 'yes' ] && [ "$COOKIE" = "$EMQX_DEFAULT_ERLANG_COOKIE" ]; then + logwarn "Default (insecure) Erlang cookie is in use." + logwarn "Configure node.cookie in $EMQX_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE" + logwarn "NOTE: Use the same cookie for all nodes in the cluster." + fi +} ## check if OTP version has mnesia_hook feature; if not, fallback to ## using Mnesia DB backend. @@ -933,10 +967,7 @@ cd "$RUNNER_ROOT_DIR" case "${COMMAND}" in start) - # Make sure a node IS not running - if relx_nodetool "ping" >/dev/null 2>&1; then - die "Node $NAME is already running!" - fi + maybe_warn_default_cookie # this flag passes down to console mode # so we know it's intended to be run in daemon mode @@ -1108,6 +1139,7 @@ case "${COMMAND}" in tr_log_to_env else maybe_log_to_console + maybe_warn_default_cookie fi #generate app.config and vm.args diff --git a/build b/build index 195464612..de00aba6c 100755 --- a/build +++ b/build @@ -1,18 +1,12 @@ #!/usr/bin/env bash # This script helps to build release artifacts. -# arg1: profile, e.g. emqx | emqx-pkg +# arg1: profile, e.g. emqx | emqx-enterprise # arg2: artifact, e.g. rel | relup | tgz | pkg -if [[ -n "$DEBUG" ]]; then - set -x -fi set -euo pipefail -DEBUG="${DEBUG:-0}" -if [ "$DEBUG" -eq 1 ]; then - set -x -fi +[ "${DEBUG:-0}" -eq 1 ] && set -x PROFILE_ARG="$1" ARTIFACT="$2" @@ -239,6 +233,9 @@ make_tgz() { macos*) target_name="${PROFILE}-${full_vsn}.zip" ;; + windows*) + target_name="${PROFILE}-${full_vsn}.zip" + ;; *) target_name="${PROFILE}-${full_vsn}.tar.gz" ;; @@ -304,6 +301,13 @@ make_tgz() { # sha256sum may not be available on macos openssl dgst -sha256 "${target}" | cut -d ' ' -f 2 > "${target}.sha256" ;; + windows*) + pushd "${tard}" >/dev/null + 7z a "${target_name}" ./emqx/* >/dev/null + popd >/dev/null + mv "${tard}/${target_name}" "${target}" + sha256sum "${target}" | head -c 64 > "${target}.sha256" + ;; *) ## create tar after change dir ## to avoid creating an extra level of 'emqx' dir in the .tar.gz file @@ -318,6 +322,12 @@ make_tgz() { log "Archive sha256sum: $(cat "${target}.sha256")" } +trap docker_cleanup EXIT + +docker_cleanup() { + rm -f ./.dockerignore >/dev/null +} + ## This function builds the default docker image based on debian 11 make_docker() { EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" @@ -329,6 +339,7 @@ make_docker() { local default_tag="emqx/${PROFILE%%-elixir}:${PKG_VSN}" EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}" + echo '_build' >> ./.dockerignore set -x docker build --no-cache --pull \ --build-arg BUILD_FROM="${EMQX_BUILDER}" \ @@ -336,6 +347,7 @@ make_docker() { --build-arg EMQX_NAME="$PROFILE" \ --tag "${EMQX_IMAGE_TAG}" \ -f "${EMQX_DOCKERFILE}" . + [[ "${DEBUG:-}" -eq 1 ]] || set +x } function join { diff --git a/changes/ce/.gitkeep b/changes/ce/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/changes/ce/feat-9213.en.md b/changes/ce/feat-9213.en.md new file mode 100644 index 000000000..3266ed836 --- /dev/null +++ b/changes/ce/feat-9213.en.md @@ -0,0 +1 @@ +Add pod disruption budget to helm chart diff --git a/changes/ce/feat-9213.zh.md b/changes/ce/feat-9213.zh.md new file mode 100644 index 000000000..509b1e01c --- /dev/null +++ b/changes/ce/feat-9213.zh.md @@ -0,0 +1 @@ +在舵手图中添加吊舱干扰预算。 diff --git a/changes/ce/feat-9949.en.md b/changes/ce/feat-9949.en.md new file mode 100644 index 000000000..3ed9c30b2 --- /dev/null +++ b/changes/ce/feat-9949.en.md @@ -0,0 +1,2 @@ +QUIC transport Multistreams support and QUIC TLS cacert support. + diff --git a/changes/ce/feat-9949.zh.md b/changes/ce/feat-9949.zh.md new file mode 100644 index 000000000..6efabac3f --- /dev/null +++ b/changes/ce/feat-9949.zh.md @@ -0,0 +1 @@ +QUIC 传输多流支持和 QUIC TLS cacert 支持。 diff --git a/changes/ce/fix-10009.en.md b/changes/ce/fix-10009.en.md new file mode 100644 index 000000000..37f33a958 --- /dev/null +++ b/changes/ce/fix-10009.en.md @@ -0,0 +1 @@ +Validate `bytes` param to `GET /trace/:name/log` to not exceed signed 32bit integer. diff --git a/changes/ce/fix-10009.zh.md b/changes/ce/fix-10009.zh.md new file mode 100644 index 000000000..bb55ea5b9 --- /dev/null +++ b/changes/ce/fix-10009.zh.md @@ -0,0 +1 @@ +验证 `GET /trace/:name/log` 的 `bytes` 参数，使其不超过有符号的32位整数。 diff --git a/changes/ce/fix-9958.en.md b/changes/ce/fix-9958.en.md new file mode 100644 index 000000000..821934ad0 --- /dev/null +++ b/changes/ce/fix-9958.en.md @@ -0,0 +1 @@ +Fix bad http response format when client ID is not found in `clients` APIs diff --git a/changes/ce/fix-9958.zh.md b/changes/ce/fix-9958.zh.md new file mode 100644 index 000000000..a26fbb7fe --- /dev/null +++ b/changes/ce/fix-9958.zh.md @@ -0,0 +1 @@ +修复 `clients` API 在 Client ID 不存在时返回的错误的 HTTP 应答格式。 diff --git a/changes/v5.0.18/fix-9961.en.md b/changes/ce/fix-9961.en.md similarity index 100% rename from changes/v5.0.18/fix-9961.en.md rename to changes/ce/fix-9961.en.md diff --git a/changes/v5.0.18/fix-9961.zh.md b/changes/ce/fix-9961.zh.md similarity index 100% rename from changes/v5.0.18/fix-9961.zh.md rename to changes/ce/fix-9961.zh.md diff --git a/changes/ce/fix-9974.en.md b/changes/ce/fix-9974.en.md new file mode 100644 index 000000000..97223e03f --- /dev/null +++ b/changes/ce/fix-9974.en.md @@ -0,0 +1,2 @@ +Report memory usage to statsd and prometheus using the same data source as dashboard. +Prior to this fix, the memory usage data source was collected from an outdated source which did not work well in containers. diff --git a/changes/ce/fix-9974.zh.md b/changes/ce/fix-9974.zh.md new file mode 100644 index 000000000..8358204f3 --- /dev/null +++ b/changes/ce/fix-9974.zh.md @@ -0,0 +1,2 @@ +Statsd 和 prometheus 使用跟 Dashboard 相同的内存用量数据源。 +在此修复前，内存的总量和用量统计使用了过时的（在容器环境中不准确）的数据源。 diff --git a/changes/ce/fix-9978.en.md b/changes/ce/fix-9978.en.md new file mode 100644 index 000000000..6750d136f --- /dev/null +++ b/changes/ce/fix-9978.en.md @@ -0,0 +1,2 @@ +Fixed configuration issue when choosing to use SSL for a Postgres connection (`authn`, `authz` and bridge). +The connection could fail to complete with a previously working configuration after an upgrade from 5.0.13 to newer EMQX versions. diff --git a/changes/ce/fix-9978.zh.md b/changes/ce/fix-9978.zh.md new file mode 100644 index 000000000..75eed3600 --- /dev/null +++ b/changes/ce/fix-9978.zh.md @@ -0,0 +1,2 @@ +修正了在Postgres连接中选择使用SSL时的配置问题（`authn`, `authz` 和 bridge）。 +从5.0.13升级到较新的EMQX版本后，连接可能无法完成之前的配置。 diff --git a/changes/ce/perf-9967.en.md b/changes/ce/perf-9967.en.md new file mode 100644 index 000000000..fadba24c9 --- /dev/null +++ b/changes/ce/perf-9967.en.md @@ -0,0 +1 @@ +New common TLS option 'hibernate_after' to reduce memory footprint per idle connecion, default: 5s. diff --git a/changes/ce/perf-9967.zh.md b/changes/ce/perf-9967.zh.md new file mode 100644 index 000000000..7b73f9bd0 --- /dev/null +++ b/changes/ce/perf-9967.zh.md @@ -0,0 +1 @@ +新的通用 TLS 选项 'hibernate_after'， 以减少空闲连接的内存占用，默认： 5s 。 diff --git a/changes/ee/.gitkeep b/changes/ee/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/changes/ee/feat-10011.en.md b/changes/ee/feat-10011.en.md new file mode 100644 index 000000000..3266ed836 --- /dev/null +++ b/changes/ee/feat-10011.en.md @@ -0,0 +1 @@ +Add pod disruption budget to helm chart diff --git a/changes/ee/feat-10011.zh.md b/changes/ee/feat-10011.zh.md new file mode 100644 index 000000000..509b1e01c --- /dev/null +++ b/changes/ee/feat-10011.zh.md @@ -0,0 +1 @@ +在舵手图中添加吊舱干扰预算。 diff --git a/changes/ee/feat-9932-en.md b/changes/ee/feat-9932-en.md new file mode 100644 index 000000000..f4f9ce40d --- /dev/null +++ b/changes/ee/feat-9932-en.md @@ -0,0 +1 @@ +Integrate `TDengine` into `bridges` as a new backend. diff --git a/changes/ee/feat-9932-zh.md b/changes/ee/feat-9932-zh.md new file mode 100644 index 000000000..1fbf7bf34 --- /dev/null +++ b/changes/ee/feat-9932-zh.md @@ -0,0 +1 @@ +在 `桥接` 中集成 `TDengine`。 diff --git a/deploy/charts/README.md b/deploy/charts/README.md new file mode 100644 index 000000000..4b8829056 --- /dev/null +++ b/deploy/charts/README.md @@ -0,0 +1,3 @@ +# Sync changes to emqx-enterprise + +When making changes in charts, please update `emqx` charts and run `./sync-enterprise.sh`. diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md index 2899dc7e0..258c9c075 100644 --- a/deploy/charts/emqx-enterprise/README.md +++ b/deploy/charts/emqx-enterprise/README.md @@ -40,7 +40,7 @@ The following table lists the configurable parameters of the emqx chart and thei | Parameter | Description | Default Value | |--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| | `replicaCount` | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split. | 3 | -| `image.repository` | EMQX Image name | `emqx/emqx-enterprise` | +| `image.repository` | EMQX Image name | emqx/emqx-enterprise | | `image.pullPolicy` | The image pull policy | IfNotPresent | | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) | | `serviceAccount.create` | If `true`, create a new service account | `true` | @@ -68,28 +68,30 @@ The following table lists the configurable parameters of the emqx chart and thei | `service.dashboard` | Port for dashboard and API. | 18083 | | `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | | `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | -| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | | `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil | | `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil | | `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil | | `service.loadBalancerIP` | loadBalancerIP for Service | nil | | `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | | `service.externalIPs` | ExternalIPs for the service | [] | -`service.externalTrafficPolicy` | External Traffic Policy for the service | `Cluster` +| `service.externalTrafficPolicy` | External Traffic Policy for the service | `Cluster` | `service.annotations` | Service annotations | {}(evaluated as a template) | | `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | | `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | | `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | | `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` | -| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | -| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | -| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | -| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | -| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `ingress.dashboard.hosts` | Ingress hosts for EMQX Dashboard | dashboard.emqx.local | +| `ingress.dashboard.tls` | Ingress tls for EMQX Dashboard | [] | +| `ingress.dashboard.annotations` | Ingress annotations for EMQX Dashboard | {} | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | +| `ingress.mqtt.enabled` | Enable ingress for MQTT | false | +| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | | +| `ingress.mqtt.path` | Ingress path for MQTT | / | +| `ingress.mqtt.pathType` | Ingress pathType for MQTT | `ImplementationSpecific` | +| `ingress.mqtt.hosts` | Ingress hosts for MQTT | mqtt.emqx.local | +| `ingress.mqtt.tls` | Ingress tls for MQTT | [] | +| `ingress.mqtt.annotations` | Ingress annotations for MQTT | {} | +| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | | | `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | | `metrics.type` | Now we only supported "prometheus" | "prometheus" | | `ssl.enabled` | Enable SSL support | false | @@ -121,3 +123,17 @@ which needs to explicitly configured by either changing the emqx config file or If you chose to use an existing certificate, make sure, you update the filenames accordingly. +## Tips +Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx. +In order to preserve the original client's IP address, you could change the emqx config by passing the following environment variable: + +``` +EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL: "true" +``` + +With haproxy you'd also need the following ingress annotation: + +``` +haproxy-ingress.github.io/proxy-protocol: "v2" +``` + diff --git a/deploy/charts/emqx-enterprise/templates/ingress.yaml b/deploy/charts/emqx-enterprise/templates/ingress.yaml index b6f496d88..29bac213d 100644 --- a/deploy/charts/emqx-enterprise/templates/ingress.yaml +++ b/deploy/charts/emqx-enterprise/templates/ingress.yaml @@ -48,3 +48,53 @@ spec: {{- end }} --- {{- end }} +{{- if .Values.ingress.mqtt.enabled -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ printf "%s-%s" (include "emqx.fullname" .) "mqtt" }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.ingress.mqtt.annotations }} + annotations: + {{- toYaml .Values.ingress.mqtt.annotations | nindent 4 }} + {{- end }} +spec: +{{- if and .Values.ingress.mqtt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.mqtt.ingressClassName }} +{{- end }} + rules: + {{- range $host := .Values.ingress.mqtt.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ $.Values.ingress.mqtt.path | default "/" }} + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ $.Values.ingress.mqtt.pathType | default "ImplementationSpecific" }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.mqtt }} + {{- else }} + serviceName: {{ include "emqx.fullname" $ }} + servicePort: {{ $.Values.service.mqtt }} + {{- end }} + {{- end -}} + {{- if .Values.ingress.mqtt.tls }} + tls: + {{- toYaml .Values.ingress.mqtt.tls | nindent 4 }} + {{- end }} +--- +{{- end }} diff --git a/deploy/charts/emqx-enterprise/templates/pdb.yaml b/deploy/charts/emqx-enterprise/templates/pdb.yaml new file mode 100644 index 000000000..a3f233064 --- /dev/null +++ b/deploy/charts/emqx-enterprise/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if and (.Values.pdb.enabled) (.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "emqx.fullname" . }}-pdb + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deploy/charts/emqx-enterprise/templates/service.yaml b/deploy/charts/emqx-enterprise/templates/service.yaml index 0fe3dc411..233e69b10 100644 --- a/deploy/charts/emqx-enterprise/templates/service.yaml +++ b/deploy/charts/emqx-enterprise/templates/service.yaml @@ -121,7 +121,7 @@ spec: port: {{ .Values.service.mqtt | default 1883 }} protocol: TCP targetPort: mqtt - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt port: {{ .Values.service.internalmqtt | default 11883 }} protocol: TCP diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index 3a607a71e..9ae863219 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -102,7 +102,7 @@ initContainers: {} # sysctl -w net.netfilter.nf_conntrack_max=1000000 # sysctl -w net.netfilter.nf_conntrack_tcp_timeout_time_wait=30 -## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx) +## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx-enterprise) emqxConfig: EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns" EMQX_DASHBOARD__DEFAULT_USERNAME: "admin" @@ -189,6 +189,20 @@ ingress: hosts: - dashboard.emqx.local tls: [] + ## ingress for MQTT + mqtt: + enabled: false + # ingressClassName: haproxy + annotations: {} + # kubernetes.io/ingress.class: haproxy + # kubernetes.io/tls-acme: "true" + # haproxy-ingress.github.io/tcp-service-port: "8883" + # haproxy-ingress.github.io/proxy-protocol: "v2" + path: / + pathType: ImplementationSpecific + hosts: + - mqtt.emqx.local + tls: [] podSecurityContext: enabled: true @@ -211,7 +225,14 @@ ssl: enabled: false useExisting: false existingName: emqx-tls - dnsnames: {} + dnsnames: [] issuer: name: letsencrypt-dns kind: ClusterIssuer + +## Setting PodDisruptionBudget. +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb +## +pdb: + enabled: false + maxUnavailable: 1 diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 6ee3617ce..e28a44199 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -68,28 +68,30 @@ The following table lists the configurable parameters of the emqx chart and thei | `service.dashboard` | Port for dashboard and API. | 18083 | | `service.nodePorts.mqtt` | Kubernetes node port for MQTT. | nil | | `service.nodePorts.mqttssl` | Kubernetes node port for MQTT(SSL). | nil | -| `service.nodePorts.mgmt` | Kubernetes node port for mgmt API. | nil | | `service.nodePorts.ws` | Kubernetes node port for WebSocket/HTTP. | nil | | `service.nodePorts.wss` | Kubernetes node port for WSS/HTTPS. | nil | | `service.nodePorts.dashboard` | Kubernetes node port for dashboard. | nil | | `service.loadBalancerIP` | loadBalancerIP for Service | nil | | `service.loadBalancerSourceRanges` | Address(es) that are allowed when service is LoadBalancer | [] | | `service.externalIPs` | ExternalIPs for the service | [] | -`service.externalTrafficPolicy` | External Traffic Policy for the service | `Cluster` +| `service.externalTrafficPolicy` | External Traffic Policy for the service | `Cluster` | `service.annotations` | Service annotations | {}(evaluated as a template) | | `ingress.dashboard.enabled` | Enable ingress for EMQX Dashboard | false | | `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | | `ingress.dashboard.path` | Ingress path for EMQX Dashboard | / | | `ingress.dashboard.pathType` | Ingress pathType for EMQX Dashboard | `ImplementationSpecific` | -| `ingress.dashboard.hosts` | Ingress hosts for EMQX Mgmt API | dashboard.emqx.local | -| `ingress.dashboard.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.dashboard.annotations` | Ingress annotations for EMQX Mgmt API | {} | -| `ingress.mgmt.enabled` | Enable ingress for EMQX Mgmt API | false | -| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API | | -| `ingress.mgmt.path` | Ingress path for EMQX Mgmt API | / | -| `ingress.mgmt.hosts` | Ingress hosts for EMQX Mgmt API | api.emqx.local | -| `ingress.mgmt.tls` | Ingress tls for EMQX Mgmt API | [] | -| `ingress.mgmt.annotations` | Ingress annotations for EMQX Mgmt API | {} | +| `ingress.dashboard.hosts` | Ingress hosts for EMQX Dashboard | dashboard.emqx.local | +| `ingress.dashboard.tls` | Ingress tls for EMQX Dashboard | [] | +| `ingress.dashboard.annotations` | Ingress annotations for EMQX Dashboard | {} | +| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard | | +| `ingress.mqtt.enabled` | Enable ingress for MQTT | false | +| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | | +| `ingress.mqtt.path` | Ingress path for MQTT | / | +| `ingress.mqtt.pathType` | Ingress pathType for MQTT | `ImplementationSpecific` | +| `ingress.mqtt.hosts` | Ingress hosts for MQTT | mqtt.emqx.local | +| `ingress.mqtt.tls` | Ingress tls for MQTT | [] | +| `ingress.mqtt.annotations` | Ingress annotations for MQTT | {} | +| `ingress.mqtt.ingressClassName` | Set the ingress class for MQTT | | | `metrics.enable` | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false | | `metrics.type` | Now we only supported "prometheus" | "prometheus" | | `ssl.enabled` | Enable SSL support | false | @@ -121,3 +123,17 @@ which needs to explicitly configured by either changing the emqx config file or If you chose to use an existing certificate, make sure, you update the filenames accordingly. +## Tips +Enable the Proxy Protocol V1/2 if the EMQX cluster is deployed behind HAProxy or Nginx. +In order to preserve the original client's IP address, you could change the emqx config by passing the following environment variable: + +``` +EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL: "true" +``` + +With haproxy you'd also need the following ingress annotation: + +``` +haproxy-ingress.github.io/proxy-protocol: "v2" +``` + diff --git a/deploy/charts/emqx/templates/ingress.yaml b/deploy/charts/emqx/templates/ingress.yaml index b6f496d88..29bac213d 100644 --- a/deploy/charts/emqx/templates/ingress.yaml +++ b/deploy/charts/emqx/templates/ingress.yaml @@ -48,3 +48,53 @@ spec: {{- end }} --- {{- end }} +{{- if .Values.ingress.mqtt.enabled -}} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ printf "%s-%s" (include "emqx.fullname" .) "mqtt" }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- if .Values.ingress.mqtt.annotations }} + annotations: + {{- toYaml .Values.ingress.mqtt.annotations | nindent 4 }} + {{- end }} +spec: +{{- if and .Values.ingress.mqtt.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.mqtt.ingressClassName }} +{{- end }} + rules: + {{- range $host := .Values.ingress.mqtt.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ $.Values.ingress.mqtt.path | default "/" }} + {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ $.Values.ingress.mqtt.pathType | default "ImplementationSpecific" }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ include "emqx.fullname" $ }} + port: + number: {{ $.Values.service.mqtt }} + {{- else }} + serviceName: {{ include "emqx.fullname" $ }} + servicePort: {{ $.Values.service.mqtt }} + {{- end }} + {{- end -}} + {{- if .Values.ingress.mqtt.tls }} + tls: + {{- toYaml .Values.ingress.mqtt.tls | nindent 4 }} + {{- end }} +--- +{{- end }} diff --git a/deploy/charts/emqx/templates/pdb.yaml b/deploy/charts/emqx/templates/pdb.yaml new file mode 100644 index 000000000..a3f233064 --- /dev/null +++ b/deploy/charts/emqx/templates/pdb.yaml @@ -0,0 +1,18 @@ +{{- if and (.Values.pdb.enabled) (.Capabilities.APIVersions.Has "policy/v1/PodDisruptionBudget") }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "emqx.fullname" . }}-pdb + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + helm.sh/chart: {{ include "emqx.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "emqx.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/deploy/charts/emqx/templates/service.yaml b/deploy/charts/emqx/templates/service.yaml index 5b6376a85..233e69b10 100644 --- a/deploy/charts/emqx/templates/service.yaml +++ b/deploy/charts/emqx/templates/service.yaml @@ -38,7 +38,7 @@ spec: {{- else if eq .Values.service.type "ClusterIP" }} nodePort: null {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }} + {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt port: {{ .Values.service.internalmqtt | default 11883 }} protocol: TCP diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index c737c8808..5f14fb17b 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -189,6 +189,20 @@ ingress: hosts: - dashboard.emqx.local tls: [] + ## ingress for MQTT + mqtt: + enabled: false + # ingressClassName: haproxy + annotations: {} + # kubernetes.io/ingress.class: haproxy + # kubernetes.io/tls-acme: "true" + # haproxy-ingress.github.io/tcp-service-port: "8883" + # haproxy-ingress.github.io/proxy-protocol: "v2" + path: / + pathType: ImplementationSpecific + hosts: + - mqtt.emqx.local + tls: [] podSecurityContext: enabled: true @@ -211,7 +225,14 @@ ssl: enabled: false useExisting: false existingName: emqx-tls - dnsnames: {} + dnsnames: [] issuer: name: letsencrypt-dns kind: ClusterIssuer + +## Setting PodDisruptionBudget. +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb +## +pdb: + enabled: false + maxUnavailable: 1 diff --git a/deploy/charts/sync-enterprise.sh b/deploy/charts/sync-enterprise.sh new file mode 100755 index 000000000..587871c0d --- /dev/null +++ b/deploy/charts/sync-enterprise.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +sed 's|emqx/emqx|emqx/emqx-enterprise|' < emqx/values.yaml > emqx-enterprise/values.yaml +cp emqx/templates/* emqx-enterprise/templates diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 4a00c68fb..f26926bce 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11 ARG RUN_FROM=debian:11-slim FROM ${BUILD_FROM} AS builder @@ -10,11 +10,9 @@ ENV EMQX_RELUP=false RUN export PROFILE=${EMQX_NAME%%-elixir} \ && export EMQX_NAME1=$EMQX_NAME \ && export EMQX_NAME=$PROFILE \ - && export EMQX_LIB_PATH="_build/$EMQX_NAME/lib" \ && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \ && export EMQX_REL_FORM='docker' \ && cd /emqx \ - && rm -rf $EMQX_LIB_PATH \ && make $EMQX_NAME1 \ && rm -f $EMQX_REL_PATH/*.tar.gz \ && mkdir -p /emqx-rel \ diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 67ae7c907..526761380 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -7,13 +7,12 @@ none := space := $(none) $(none) ## RPM does not allow '-' in version number and release string, replace with '_' RPM_VSN := $(subst -,_,$(PKG_VSN)) -RPM_REL := otp$(subst -,_,$(OTP_VSN)) EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz -SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +SOURCE_PKG := emqx-$(RPM_VSN).$(shell uname -m) TARGET_PKG := $(EMQX_NAME)-$(shell $(EMQX_REL)/pkg-vsn.sh $(EMQX_NAME) --long) # Not $(PWD) as it does not work for make -C @@ -33,7 +32,6 @@ all: | $(BUILT) --define "_topdir $(TOPDIR)" \ --define "_version $(RPM_VSN)" \ --define "_reldir $(SRCDIR)" \ - --define "_release $(RPM_REL)" \ --define "_post_addition $(POST_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \ --define "_sharedstatedir /var/lib" \ diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 366f85396..b2b58ac23 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -5,12 +5,12 @@ %define _log_dir %{_var}/log/%{_name} %define _lib_home /usr/lib/%{_name} %define _var_home %{_sharedstatedir}/%{_name} -%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm +%define _build_name_fmt %{_arch}/%{_name}-%{_version}.%{_arch}.rpm %define _build_id_links none Name: %{_package_name} Version: %{_version} -Release: %{_release}%{?dist} +Release: 1%{?dist} Summary: emqx Group: System Environment/Daemons License: Apache License Version 2.0 diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct index bf990bd7c..967faa343 100644 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ b/lib-ee/emqx_ee_bridge/docker-ct @@ -7,3 +7,4 @@ mysql redis redis_cluster pgsql +tdengine diff --git a/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf new file mode 100644 index 000000000..2d5af9f16 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/i18n/emqx_ee_bridge_tdengine.conf @@ -0,0 +1,74 @@ +emqx_ee_bridge_tdengine { + + local_topic { + desc { + en: """The MQTT topic filter to be forwarded to TDengine. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded. +""" + zh: """发送到 'local_topic' 的消息都会转发到 TDengine。
+注意：如果这个 Bridge 被用作规则（EMQX 规则引擎）的输出，同时也配置了 'local_topic' ，那么这两部分的消息都会被转发。 +""" + } + label { + en: "Local Topic" + zh: "本地 Topic" + } + } + + sql_template { + desc { + en: """SQL Template""" + zh: """SQL 模板""" + } + label { + en: "SQL Template" + zh: "SQL 模板" + } + } + config_enable { + desc { + en: """Enable or disable this bridge""" + zh: """启用/禁用桥接""" + } + label { + en: "Enable Or Disable Bridge" + zh: "启用/禁用桥接" + } + } + + desc_config { + desc { + en: """Configuration for an TDengine bridge.""" + zh: """TDengine 桥接配置""" + } + label: { + en: "TDengine Bridge Configuration" + zh: "TDengine 桥接配置" + } + } + + desc_type { + desc { + en: """The Bridge Type""" + zh: """Bridge 类型""" + } + label { + en: "Bridge Type" + zh: "桥接类型" + } + } + + desc_name { + desc { + en: """Bridge name.""" + zh: """桥接名字""" + } + label { + en: "Bridge Name" + zh: "桥接名字" + } + } +} diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src index b2a3c80c6..c30c927f2 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src @@ -1,6 +1,6 @@ {application, emqx_ee_bridge, [ {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 43a26111a..1a358fdfe 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -28,7 +28,8 @@ api_schemas(Method) -> ref(emqx_ee_bridge_redis, Method ++ "_sentinel"), ref(emqx_ee_bridge_redis, Method ++ "_cluster"), ref(emqx_ee_bridge_timescale, Method), - ref(emqx_ee_bridge_matrix, Method) + ref(emqx_ee_bridge_matrix, Method), + ref(emqx_ee_bridge_tdengine, Method) ]. schema_modules() -> @@ -42,7 +43,8 @@ schema_modules() -> emqx_ee_bridge_redis, emqx_ee_bridge_pgsql, emqx_ee_bridge_timescale, - emqx_ee_bridge_matrix + emqx_ee_bridge_matrix, + emqx_ee_bridge_tdengine ]. examples(Method) -> @@ -72,7 +74,8 @@ resource_type(redis_sentinel) -> emqx_ee_connector_redis; resource_type(redis_cluster) -> emqx_ee_connector_redis; resource_type(pgsql) -> emqx_connector_pgsql; resource_type(timescale) -> emqx_connector_pgsql; -resource_type(matrix) -> emqx_connector_pgsql. +resource_type(matrix) -> emqx_connector_pgsql; +resource_type(tdengine) -> emqx_ee_connector_tdengine. fields(bridges) -> [ @@ -107,6 +110,14 @@ fields(bridges) -> desc => <<"MySQL Bridge Config">>, required => false } + )}, + {tdengine, + mk( + hoconsc:map(name, ref(emqx_ee_bridge_tdengine, "config")), + #{ + desc => <<"TDengine Bridge Config">>, + required => false + } )} ] ++ mongodb_structs() ++ influxdb_structs() ++ redis_structs() ++ pgsql_structs(). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl index 1bee9e789..e00483839 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl @@ -50,7 +50,7 @@ fields(bridge_config) -> sc( emqx_schema:duration_ms(), #{ - default => "15s", + default => <<"15s">>, desc => ?DESC("connect_timeout") } )}, @@ -84,7 +84,7 @@ fields(bridge_config) -> emqx_schema:duration_ms(), #{ required => false, - default => "15s", + default => <<"15s">>, desc => ?DESC("request_timeout") } )}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl index 6d96e3883..14f53b5e7 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl @@ -31,12 +31,6 @@ conn_bridge_examples(Method) -> [ - #{ - <<"influxdb_udp">> => #{ - summary => <<"InfluxDB UDP Bridge">>, - value => values("influxdb_udp", Method) - } - }, #{ <<"influxdb_api_v1">> => #{ summary => <<"InfluxDB HTTP API V1 Bridge">>, @@ -71,12 +65,6 @@ values("influxdb_api_v1", post) -> server => <<"127.0.0.1:8086">> }, values(common, "influxdb_api_v1", SupportUint, TypeOpts); -values("influxdb_udp", post) -> - SupportUint = <<>>, - TypeOpts = #{ - server => <<"127.0.0.1:8089">> - }, - values(common, "influxdb_udp", SupportUint, TypeOpts); values(Protocol, put) -> values(Protocol, post). @@ -106,26 +94,20 @@ namespace() -> "bridge_influxdb". roots() -> []. -fields("post_udp") -> - method_fileds(post, influxdb_udp); fields("post_api_v1") -> method_fileds(post, influxdb_api_v1); fields("post_api_v2") -> method_fileds(post, influxdb_api_v2); -fields("put_udp") -> - method_fileds(put, influxdb_udp); fields("put_api_v1") -> method_fileds(put, influxdb_api_v1); fields("put_api_v2") -> method_fileds(put, influxdb_api_v2); -fields("get_udp") -> - method_fileds(get, influxdb_udp); fields("get_api_v1") -> method_fileds(get, influxdb_api_v1); fields("get_api_v2") -> method_fileds(get, influxdb_api_v2); fields(Type) when - Type == influxdb_udp orelse Type == influxdb_api_v1 orelse Type == influxdb_api_v2 + Type == influxdb_api_v1 orelse Type == influxdb_api_v2 -> influxdb_bridge_common_fields() ++ connector_fields(Type). @@ -164,8 +146,6 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for InfluxDB using `", string:to_upper(Method), "` method."]; -desc(influxdb_udp) -> - ?DESC(emqx_ee_connector_influxdb, "influxdb_udp"); desc(influxdb_api_v1) -> ?DESC(emqx_ee_connector_influxdb, "influxdb_api_v1"); desc(influxdb_api_v2) -> diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl index e694f6c15..3983b235c 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl @@ -84,20 +84,20 @@ fields("config") -> )}, {connect_timeout, mk(emqx_schema:duration_ms(), #{ - default => "5s", + default => <<"5s">>, desc => ?DESC(connect_timeout) })}, {min_metadata_refresh_interval, mk( emqx_schema:duration_ms(), #{ - default => "3s", + default => <<"3s">>, desc => ?DESC(min_metadata_refresh_interval) } )}, {metadata_request_timeout, mk(emqx_schema:duration_ms(), #{ - default => "5s", + default => <<"5s">>, desc => ?DESC(metadata_request_timeout) })}, {authentication, @@ -141,12 +141,12 @@ fields(socket_opts) -> {sndbuf, mk( emqx_schema:bytesize(), - #{default => "1024KB", desc => ?DESC(socket_send_buffer)} + #{default => <<"1024KB">>, desc => ?DESC(socket_send_buffer)} )}, {recbuf, mk( emqx_schema:bytesize(), - #{default => "1024KB", desc => ?DESC(socket_receive_buffer)} + #{default => <<"1024KB">>, desc => ?DESC(socket_receive_buffer)} )}, {nodelay, mk( @@ -170,7 +170,7 @@ fields(producer_kafka_opts) -> {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})}, {message, mk(ref(kafka_message), #{required => false, desc => ?DESC(kafka_message)})}, {max_batch_bytes, - mk(emqx_schema:bytesize(), #{default => "896KB", desc => ?DESC(max_batch_bytes)})}, + mk(emqx_schema:bytesize(), #{default => <<"896KB">>, desc => ?DESC(max_batch_bytes)})}, {compression, mk(enum([no_compression, snappy, gzip]), #{ default => no_compression, desc => ?DESC(compression) @@ -192,7 +192,7 @@ fields(producer_kafka_opts) -> mk( emqx_schema:duration_s(), #{ - default => "60s", + default => <<"60s">>, desc => ?DESC(partition_count_refresh_interval) } )}, @@ -212,11 +212,11 @@ fields(producer_kafka_opts) -> ]; fields(kafka_message) -> [ - {key, mk(string(), #{default => "${.clientid}", desc => ?DESC(kafka_message_key)})}, - {value, mk(string(), #{default => "${.}", desc => ?DESC(kafka_message_value)})}, + {key, mk(string(), #{default => <<"${.clientid}">>, desc => ?DESC(kafka_message_key)})}, + {value, mk(string(), #{default => <<"${.}">>, desc => ?DESC(kafka_message_value)})}, {timestamp, mk(string(), #{ - default => "${.timestamp}", desc => ?DESC(kafka_message_timestamp) + default => <<"${.timestamp}">>, desc => ?DESC(kafka_message_timestamp) })} ]; fields(producer_buffer) -> @@ -229,12 +229,12 @@ fields(producer_buffer) -> {per_partition_limit, mk( emqx_schema:bytesize(), - #{default => "2GB", desc => ?DESC(buffer_per_partition_limit)} + #{default => <<"2GB">>, desc => ?DESC(buffer_per_partition_limit)} )}, {segment_bytes, mk( emqx_schema:bytesize(), - #{default => "100MB", desc => ?DESC(buffer_segment_bytes)} + #{default => <<"100MB">>, desc => ?DESC(buffer_segment_bytes)} )}, {memory_overload_protection, mk(boolean(), #{ diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl new file mode 100644 index 000000000..35e81efa3 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl @@ -0,0 +1,123 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_ee_bridge_tdengine). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1, + values/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(DEFAULT_SQL, << + "insert into mqtt.t_mqtt_msg(ts, msgid, mqtt_topic, qos, payload, arrived) " + "values (${ts}, ${id}, ${topic}, ${qos}, ${payload}, ${timestamp})" +>>). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_examples(Method) -> + [ + #{ + <<"tdengine">> => #{ + summary => <<"TDengine Bridge">>, + value => values(Method) + } + } + ]. + +values(get) -> + maps:merge(values(post), ?METRICS_EXAMPLE); +values(post) -> + #{ + enable => true, + type => tdengine, + name => <<"foo">>, + server => <<"127.0.0.1:6041">>, + database => <<"mqtt">>, + pool_size => 8, + username => <<"root">>, + password => <<"taosdata">>, + sql => ?DEFAULT_SQL, + local_topic => <<"local/topic/#">>, + resource_opts => #{ + worker_pool_size => 8, + health_check_interval => ?HEALTHCHECK_INTERVAL_RAW, + auto_restart_interval => ?AUTO_RESTART_INTERVAL_RAW, + batch_size => ?DEFAULT_BATCH_SIZE, + batch_time => ?DEFAULT_BATCH_TIME, + query_mode => sync, + max_queue_bytes => ?DEFAULT_QUEUE_SIZE + } + }; +values(put) -> + values(post). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_tdengine". + +roots() -> []. + +fields("config") -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {sql, + mk( + binary(), + #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>} + )}, + {local_topic, + mk( + binary(), + #{desc => ?DESC("local_topic"), default => undefined} + )}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ] ++ emqx_ee_connector_tdengine:fields(config); +fields("creation_opts") -> + emqx_resource_schema:fields("creation_opts_sync_only"); +fields("post") -> + [type_field(), name_field() | fields("config")]; +fields("put") -> + fields("config"); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for TDengine using `", string:to_upper(Method), "` method."]; +desc("creation_opts" = Name) -> + emqx_resource_schema:desc(Name); +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- + +type_field() -> + {type, mk(enum([tdengine]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl index ac98209ed..cff17b7de 100644 --- a/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl +++ b/lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl @@ -228,7 +228,7 @@ render_timestamp(Template, Message) -> %% Wolff producer never gives up retrying %% so there can only be 'ok' results. on_kafka_ack(_Partition, Offset, {ReplyFn, Args}) when is_integer(Offset) -> - %% the ReplyFn is emqx_resource_worker:handle_async_reply/2 + %% the ReplyFn is emqx_resource_buffer_worker:handle_async_reply/2 apply(ReplyFn, Args ++ [ok]); on_kafka_ack(_Partition, buffer_overflow_discarded, _Callback) -> %% wolff should bump the dropped_queue_full counter diff --git a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl index 17484b948..d06218397 100644 --- a/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl +++ b/lib-ee/emqx_ee_bridge/test/emqx_bridge_impl_kafka_producer_SUITE.erl @@ -277,6 +277,9 @@ kafka_bridge_rest_api_helper(Config) -> }, <<"kafka">> => #{ <<"topic">> => erlang:list_to_binary(KafkaTopic), + <<"buffer">> => #{ + <<"memory_overload_protection">> => <<"false">> + }, <<"message">> => #{ <<"key">> => <<"${clientid}">>, <<"value">> => <<"${.payload}">> @@ -384,6 +387,13 @@ t_failed_creation_then_fix(Config) -> "kafka_hosts_string" => HostsString, "kafka_topic" => KafkaTopic, "instance_id" => ResourceId, + "producer" => #{ + "kafka" => #{ + "buffer" => #{ + "memory_overload_protection" => false + } + } + }, "ssl" => #{} }), %% creates, but fails to start producers @@ -577,6 +587,9 @@ producer = { topic = \"{{ kafka_topic }}\" message = {key = \"${clientid}\", value = \"${.payload}\"} partition_strategy = {{ partition_strategy }} + buffer = { + memory_overload_protection = false + } } } """. diff --git a/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl new file mode 100644 index 000000000..4c17ba1a1 --- /dev/null +++ b/lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_tdengine_SUITE.erl @@ -0,0 +1,432 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_bridge_tdengine_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% SQL definitions +-define(SQL_BRIDGE, + "insert into mqtt.t_mqtt_msg(ts, payload) values (${timestamp}, ${payload})" +). + +-define(SQL_CREATE_DATABASE, "CREATE DATABASE IF NOT EXISTS mqtt; USE mqtt;"). +-define(SQL_CREATE_TABLE, + "CREATE TABLE t_mqtt_msg (\n" + " ts timestamp,\n" + " payload BINARY(1024)\n" + ");" +). +-define(SQL_DROP_TABLE, "DROP TABLE t_mqtt_msg"). +-define(SQL_DELETE, "DELETE from t_mqtt_msg"). +-define(SQL_SELECT, "SELECT payload FROM t_mqtt_msg"). + +% DB defaults +-define(TD_DATABASE, "mqtt"). +-define(TD_USERNAME, "root"). +-define(TD_PASSWORD, "taosdata"). +-define(BATCH_SIZE, 10). +-define(PAYLOAD, <<"HELLO">>). + +-define(WITH_CON(Process), + Con = connect_direct_tdengine(Config), + Process, + ok = tdengine:stop(Con) +). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, with_batch}, + {group, without_batch} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + NonBatchCases = [t_write_timeout], + [ + {with_batch, TCs -- NonBatchCases}, + {without_batch, TCs} + ]. + +init_per_group(with_batch, Config0) -> + Config = [{enable_batch, true} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{enable_batch, false} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> + connect_and_drop_table(Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok. + +init_per_testcase(_Testcase, Config) -> + connect_and_clear_table(Config), + delete_bridge(Config), + Config. + +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + connect_and_clear_table(Config), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(ConfigT) -> + Host = os:getenv("TDENGINE_HOST", "toxiproxy"), + Port = list_to_integer(os:getenv("TDENGINE_PORT", "6041")), + + Config0 = [ + {td_host, Host}, + {td_port, Port}, + {query_mode, sync}, + {proxy_name, "tdengine_restful"} + | ConfigT + ], + + BridgeType = proplists:get_value(bridge_type, Config0, <<"tdengine">>), + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + _ = application:load(emqx_ee_bridge), + _ = emqx_ee_bridge:module_info(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + emqx_mgmt_api_test_util:init_suite(), + % Connect to tdengine directly and create the table + connect_and_create_table(Config0), + {Name, TDConf} = tdengine_config(BridgeType, Config0), + Config = + [ + {tdengine_config, TDConf}, + {tdengine_bridge_type, BridgeType}, + {tdengine_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_tdengine); + _ -> + {skip, no_tdengine} + end + end. + +tdengine_config(BridgeType, Config) -> + Port = integer_to_list(?config(td_port, Config)), + Server = ?config(td_host, Config) ++ ":" ++ Port, + Name = atom_to_binary(?MODULE), + BatchSize = + case ?config(enable_batch, Config) of + true -> ?BATCH_SIZE; + false -> 1 + end, + QueryMode = ?config(query_mode, Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " server = ~p\n" + " database = ~p\n" + " username = ~p\n" + " password = ~p\n" + " sql = ~p\n" + " resource_opts = {\n" + " request_timeout = 500ms\n" + " batch_size = ~b\n" + " query_mode = ~s\n" + " }\n" + "}", + [ + BridgeType, + Name, + Server, + ?TD_DATABASE, + ?TD_USERNAME, + ?TD_PASSWORD, + ?SQL_BRIDGE, + BatchSize, + QueryMode + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +create_bridge(Config) -> + BridgeType = ?config(tdengine_bridge_type, Config), + Name = ?config(tdengine_name, Config), + TDConfig = ?config(tdengine_config, Config), + emqx_bridge:create(BridgeType, Name, TDConfig). + +delete_bridge(Config) -> + BridgeType = ?config(tdengine_bridge_type, Config), + Name = ?config(tdengine_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Payload) -> + Name = ?config(tdengine_name, Config), + BridgeType = ?config(tdengine_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Payload). + +query_resource(Config, Request) -> + Name = ?config(tdengine_name, Config), + BridgeType = ?config(tdengine_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). + +connect_direct_tdengine(Config) -> + Opts = [ + {host, to_bin(?config(td_host, Config))}, + {port, ?config(td_port, Config)}, + {username, to_bin(?TD_USERNAME)}, + {password, to_bin(?TD_PASSWORD)}, + {pool_size, 8} + ], + + {ok, Con} = tdengine:start_link(Opts), + Con. + +% These funs connect and then stop the tdengine connection +connect_and_create_table(Config) -> + ?WITH_CON(begin + {ok, _} = directly_query(Con, ?SQL_CREATE_DATABASE, []), + {ok, _} = directly_query(Con, ?SQL_CREATE_TABLE) + end). + +connect_and_drop_table(Config) -> + ?WITH_CON({ok, _} = directly_query(Con, ?SQL_DROP_TABLE)). + +connect_and_clear_table(Config) -> + ?WITH_CON({ok, _} = directly_query(Con, ?SQL_DELETE)). + +connect_and_get_payload(Config) -> + ?WITH_CON( + {ok, #{<<"code">> := 0, <<"data">> := [[Result]]}} = directly_query(Con, ?SQL_SELECT) + ), + Result. + +directly_query(Con, Query) -> + directly_query(Con, Query, [{db_name, ?TD_DATABASE}]). + +directly_query(Con, Query, QueryOpts) -> + tdengine:insert(Con, Query, QueryOpts). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, + ?check_trace( + begin + ?wait_async_action( + ?assertMatch( + {ok, #{<<"code">> := 0, <<"rows">> := 1}}, send_message(Config, SentData) + ), + #{?snk_kind := tdengine_connector_query_return}, + 10_000 + ), + ?assertMatch( + ?PAYLOAD, + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(tdengine_connector_query_return, Trace0), + ?assertMatch([#{result := {ok, #{<<"code">> := 0, <<"rows">> := 1}}}], Trace), + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?config(tdengine_bridge_type, Config), + Name = ?config(tdengine_name, Config), + PgsqlConfig0 = ?config(tdengine_config, Config), + PgsqlConfig = PgsqlConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(PgsqlConfig) + ), + SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, + ?check_trace( + begin + ?wait_async_action( + ?assertMatch( + {ok, #{<<"code">> := 0, <<"rows">> := 1}}, send_message(Config, SentData) + ), + #{?snk_kind := tdengine_connector_query_return}, + 10_000 + ), + ?assertMatch( + ?PAYLOAD, + connect_and_get_payload(Config) + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(tdengine_connector_query_return, Trace0), + ?assertMatch([#{result := {ok, #{<<"code">> := 0, <<"rows">> := 1}}}], Trace), + ok + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + Name = ?config(tdengine_name, Config), + BridgeType = ?config(tdengine_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {ok, Status} when Status =:= disconnected orelse Status =:= connecting, + emqx_resource_manager:health_check(ResourceID) + ) + end), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + {ok, _} = create_bridge(Config), + SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({error, econnrefused}, send_message(Config, SentData)) + end), + ok. + +% This test doesn't work with batch enabled since it is not possible +% to set the timeout directly for batch queries +t_write_timeout(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + {ok, _} = create_bridge(Config), + SentData = #{payload => ?PAYLOAD, timestamp => 1668602148000}, + emqx_common_test_helpers:with_failure(timeout, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + query_resource(Config, {send_message, SentData}) + ) + end), + ok. + +t_simple_sql_query(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Request = {query, <<"SELECT count(1) AS T">>}, + Result = query_resource(Config, Request), + case ?config(enable_batch, Config) of + true -> + ?assertEqual({error, {unrecoverable_error, batch_prepare_not_implemented}}, Result); + false -> + ?assertMatch({ok, #{<<"code">> := 0, <<"data">> := [[1]]}}, Result) + end, + ok. + +t_missing_data(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Result = send_message(Config, #{}), + ?assertMatch( + {error, #{ + <<"code">> := 534, + <<"desc">> := _ + }}, + Result + ), + ok. + +t_bad_sql_parameter(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Request = {sql, <<"">>, [bad_parameter]}, + Result = query_resource(Config, Request), + case ?config(enable_batch, Config) of + true -> + ?assertEqual({error, {unrecoverable_error, invalid_request}}, Result); + false -> + ?assertMatch( + {error, {unrecoverable_error, _}}, Result + ) + end, + ok. + +to_bin(List) when is_list(List) -> + unicode:characters_to_binary(List, utf8); +to_bin(Bin) when is_binary(Bin) -> + Bin. diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct new file mode 100644 index 000000000..ef579c036 --- /dev/null +++ b/lib-ee/emqx_ee_connector/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +influxdb diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf index c00e88ef9..18ff48109 100644 --- a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_influxdb.conf @@ -26,24 +26,14 @@ The InfluxDB default port 8086 is used if `[:Port]` is not specified.""" } protocol { desc { - en: """InfluxDB's protocol. UDP or HTTP API or HTTP API V2.""" - zh: """InfluxDB 协议。UDP 或 HTTP API 或 HTTP API V2。""" + en: """InfluxDB's protocol. HTTP API or HTTP API V2.""" + zh: """InfluxDB 协议。HTTP API 或 HTTP API V2。""" } label { en: """Protocol""" zh: """协议""" } } - influxdb_udp { - desc { - en: """InfluxDB's UDP protocol.""" - zh: """InfluxDB UDP 协议。""" - } - label { - en: """UDP Protocol""" - zh: """UDP 协议""" - } - } influxdb_api_v1 { desc { en: """InfluxDB's protocol. Support InfluxDB v1.8 and before.""" diff --git a/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf new file mode 100644 index 000000000..c6c58d82d --- /dev/null +++ b/lib-ee/emqx_ee_connector/i18n/emqx_ee_connector_tdengine.conf @@ -0,0 +1,22 @@ +emqx_ee_connector_tdengine { + + server { + desc { + en: """ +The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
+The TDengine default port 6041 is used if `[:Port]` is not specified. +""" + zh: """ +将要连接的 IPv4 或 IPv6 地址，或者主机名。
+主机名具有以下形式：`Host[:Port]`。
+如果未指定 `[:Port]`，则使用 TDengine 默认端口 6041。 +""" + } + label: { + en: "Server Host" + zh: "服务器地址" + } + } + +} diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index 00421e4f6..54c471f96 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -2,6 +2,7 @@ {deps, [ {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.8"}}}, + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.5"}}}, {emqx, {path, "../../apps/emqx"}} ]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src index 56d128601..5017abd21 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src @@ -1,12 +1,13 @@ {application, emqx_ee_connector, [ {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, stdlib, hstreamdb_erl, influxdb, + tdengine, wolff, brod ]}, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl index d689f4bf3..785ec5d07 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_influxdb.erl @@ -29,6 +29,7 @@ -export([reply_callback/2]). -export([ + roots/0, namespace/0, fields/1, desc/1 @@ -139,6 +140,18 @@ on_get_status(_InstId, #{client := Client}) -> %% schema namespace() -> connector_influxdb. +roots() -> + [ + {config, #{ + type => hoconsc:union( + [ + hoconsc:ref(?MODULE, influxdb_api_v1), + hoconsc:ref(?MODULE, influxdb_api_v2) + ] + ) + }} + ]. + fields(common) -> [ {server, server()}, @@ -151,8 +164,6 @@ fields(common) -> required => false, default => ms, desc => ?DESC("precision") })} ]; -fields(influxdb_udp) -> - fields(common); fields(influxdb_api_v1) -> fields(common) ++ [ @@ -185,8 +196,6 @@ server() -> desc(common) -> ?DESC("common"); -desc(influxdb_udp) -> - ?DESC("influxdb_udp"); desc(influxdb_api_v1) -> ?DESC("influxdb_api_v1"); desc(influxdb_api_v2) -> @@ -312,12 +321,7 @@ protocol_config(#{ {bucket, str(Bucket)}, {org, str(Org)}, {token, Token} - ] ++ ssl_config(SSL); -%% udp config -protocol_config(_) -> - [ - {protocol, udp} - ]. + ] ++ ssl_config(SSL). ssl_config(#{enable := false}) -> [ @@ -327,7 +331,7 @@ ssl_config(SSL = #{enable := true}) -> [ {https_enabled, true}, {transport, ssl}, - {transport_opts, maps:to_list(maps:remove(enable, SSL))} + {transport_opts, emqx_tls_lib:to_client_opts(SSL)} ]. username(#{username := Username}) -> @@ -645,10 +649,6 @@ desc_test_() -> {desc, _, _}, desc(common) ), - ?_assertMatch( - {desc, _, _}, - desc(influxdb_udp) - ), ?_assertMatch( {desc, _, _}, desc(influxdb_api_v1) diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl new file mode 100644 index 000000000..7ab0c5078 --- /dev/null +++ b/lib-ee/emqx_ee_connector/src/emqx_ee_connector_tdengine.erl @@ -0,0 +1,241 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_tdengine). + +-behaviour(emqx_resource). + +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-export([roots/0, fields/1]). + +%% `emqx_resource' API +-export([ + callback_mode/0, + is_buffer_supported/0, + on_start/2, + on_stop/2, + on_query/3, + on_batch_query/3, + on_get_status/2 +]). + +-export([connect/1, do_get_status/1, execute/3]). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-define(TD_HOST_OPTIONS, #{ + default_port => 6041 +}). + +%%===================================================================== +%% Hocon schema +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + [ + {server, server()} + | add_default_username(emqx_connector_schema_lib:relational_db_fields()) + ]. + +add_default_username(Fields) -> + lists:map( + fun + ({username, OrigUsernameFn}) -> + {username, add_default_fn(OrigUsernameFn, <<"root">>)}; + (Field) -> + Field + end, + Fields + ). + +add_default_fn(OrigFn, Default) -> + fun + (default) -> Default; + (Field) -> OrigFn(Field) + end. + +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?TD_HOST_OPTIONS). + +%%======================================================================================== +%% `emqx_resource' API +%%======================================================================================== + +callback_mode() -> always_sync. + +is_buffer_supported() -> false. + +on_start( + InstanceId, + #{ + server := Server, + username := Username, + password := Password, + pool_size := PoolSize + } = Config +) -> + ?SLOG(info, #{ + msg => "starting_tdengine_connector", + connector => InstanceId, + config => emqx_misc:redact(Config) + }), + + {Host, Port} = emqx_schema:parse_server(Server, ?TD_HOST_OPTIONS), + Options = [ + {host, to_bin(Host)}, + {port, Port}, + {username, Username}, + {password, Password}, + {pool_size, PoolSize}, + {pool, binary_to_atom(InstanceId, utf8)} + ], + + Prepares = parse_prepare_sql(Config), + State = maps:merge(Prepares, #{poolname => InstanceId, query_opts => query_opts(Config)}), + case emqx_plugin_libs_pool:start_pool(InstanceId, ?MODULE, Options) of + ok -> + {ok, State}; + Error -> + Error + end. + +on_stop(InstanceId, #{poolname := PoolName} = _State) -> + ?SLOG(info, #{ + msg => "stopping_tdengine_connector", + connector => InstanceId + }), + emqx_plugin_libs_pool:stop_pool(PoolName). + +on_query(InstanceId, {query, SQL}, State) -> + do_query(InstanceId, SQL, State); +on_query(InstanceId, Request, State) -> + %% because the `emqx-tdengine` client only supports a single SQL cmd + %% so the `on_query` and `on_batch_query` have the same process, that is: + %% we need to collect all data into one SQL cmd and then call the insert API + on_batch_query(InstanceId, [Request], State). + +on_batch_query( + InstanceId, + BatchReq, + #{batch_inserts := Inserts, batch_params_tokens := ParamsTokens} = State +) -> + case hd(BatchReq) of + {Key, _} -> + case maps:get(Key, Inserts, undefined) of + undefined -> + {error, {unrecoverable_error, batch_prepare_not_implemented}}; + InsertSQL -> + Tokens = maps:get(Key, ParamsTokens), + do_batch_insert(InstanceId, BatchReq, InsertSQL, Tokens, State) + end; + Request -> + LogMeta = #{connector => InstanceId, first_request => Request, state => State}, + ?SLOG(error, LogMeta#{msg => "invalid request"}), + {error, {unrecoverable_error, invalid_request}} + end. + +on_get_status(_InstanceId, #{poolname := Pool}) -> + Health = emqx_plugin_libs_pool:health_check_ecpool_workers(Pool, fun ?MODULE:do_get_status/1), + status_result(Health). + +do_get_status(Conn) -> + case tdengine:insert(Conn, "select server_version()", []) of + {ok, _} -> true; + _ -> false + end. + +status_result(_Status = true) -> connected; +status_result(_Status = false) -> connecting. + +%%======================================================================================== +%% Helper fns +%%======================================================================================== + +do_batch_insert(InstanceId, BatchReqs, InsertPart, Tokens, State) -> + SQL = emqx_plugin_libs_rule:proc_batch_sql(BatchReqs, InsertPart, Tokens), + do_query(InstanceId, SQL, State). + +do_query(InstanceId, Query, #{poolname := PoolName, query_opts := Opts} = State) -> + ?TRACE( + "QUERY", + "tdengine_connector_received", + #{connector => InstanceId, query => Query, state => State} + ), + Result = ecpool:pick_and_do(PoolName, {?MODULE, execute, [Query, Opts]}, no_handover), + + case Result of + {error, Reason} -> + ?tp( + tdengine_connector_query_return, + #{error => Reason} + ), + ?SLOG(error, #{ + msg => "tdengine_connector_do_query_failed", + connector => InstanceId, + query => Query, + reason => Reason + }), + Result; + _ -> + ?tp( + tdengine_connector_query_return, + #{result => Result} + ), + Result + end. + +execute(Conn, Query, Opts) -> + tdengine:insert(Conn, Query, Opts). + +connect(Opts) -> + tdengine:start_link(Opts). + +query_opts(#{database := Database} = _Opts) -> + [{db_name, Database}]. + +parse_prepare_sql(Config) -> + SQL = + case maps:get(sql, Config, undefined) of + undefined -> #{}; + Template -> #{send_message => Template} + end, + + parse_batch_prepare_sql(maps:to_list(SQL), #{}, #{}). + +parse_batch_prepare_sql([{Key, H} | T], BatchInserts, BatchTks) -> + case emqx_plugin_libs_rule:detect_sql_type(H) of + {ok, select} -> + parse_batch_prepare_sql(T, BatchInserts, BatchTks); + {ok, insert} -> + case emqx_plugin_libs_rule:split_insert_sql(H) of + {ok, {InsertSQL, Params}} -> + ParamsTks = emqx_plugin_libs_rule:preproc_tmpl(Params), + parse_batch_prepare_sql( + T, + BatchInserts#{Key => InsertSQL}, + BatchTks#{Key => ParamsTks} + ); + {error, Reason} -> + ?SLOG(error, #{msg => "split sql failed", sql => H, reason => Reason}), + parse_batch_prepare_sql(T, BatchInserts, BatchTks) + end; + {error, Reason} -> + ?SLOG(error, #{msg => "detect sql type failed", sql => H, reason => Reason}), + parse_batch_prepare_sql(T, BatchInserts, BatchTks) + end; +parse_batch_prepare_sql([], BatchInserts, BatchTks) -> + #{ + batch_inserts => BatchInserts, + batch_params_tokens => BatchTks + }. + +to_bin(List) when is_list(List) -> + unicode:characters_to_binary(List, utf8). diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl new file mode 100644 index 000000000..f5e43c0bb --- /dev/null +++ b/lib-ee/emqx_ee_connector/test/emqx_ee_connector_influxdb_SUITE.erl @@ -0,0 +1,231 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ee_connector_influxdb_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_connector.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(INFLUXDB_RESOURCE_MOD, emqx_ee_connector_influxdb). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + InfluxDBTCPHost = os:getenv("INFLUXDB_APIV2_TCP_HOST", "toxiproxy"), + InfluxDBTCPPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TCP_PORT", "8086")), + InfluxDBTLSHost = os:getenv("INFLUXDB_APIV2_TLS_HOST", "toxiproxy"), + InfluxDBTLSPort = list_to_integer(os:getenv("INFLUXDB_APIV2_TLS_PORT", "8087")), + Servers = [{InfluxDBTCPHost, InfluxDBTCPPort}, {InfluxDBTLSHost, InfluxDBTLSPort}], + case emqx_common_test_helpers:is_all_tcp_servers_available(Servers) of + true -> + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + [ + {influxdb_tcp_host, InfluxDBTCPHost}, + {influxdb_tcp_port, InfluxDBTCPPort}, + {influxdb_tls_host, InfluxDBTLSHost}, + {influxdb_tls_port, InfluxDBTLSPort} + | Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_influxdb); + _ -> + {skip, no_influxdb} + end + end. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), + _ = application:stop(emqx_connector). + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _Config) -> + ok. + +% %%------------------------------------------------------------------------------ +% %% Testcases +% %%------------------------------------------------------------------------------ + +t_lifecycle(Config) -> + Host = ?config(influxdb_tcp_host, Config), + Port = ?config(influxdb_tcp_port, Config), + perform_lifecycle_check( + <<"emqx_ee_connector_influxdb_SUITE">>, + influxdb_config(Host, Port, false, <<"verify_none">>) + ). + +perform_lifecycle_check(PoolName, InitialConfig) -> + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(?INFLUXDB_RESOURCE_MOD, InitialConfig), + % We need to add a write_syntax to the config since the connector + % expects this + FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()}, + {ok, #{ + state := #{client := #{pool := ReturnedPoolName}} = State, + status := InitialStatus + }} = emqx_resource:create_local( + PoolName, + ?CONNECTOR_RESOURCE_GROUP, + ?INFLUXDB_RESOURCE_MOD, + FullConfig, + #{} + ), + ?assertEqual(InitialStatus, connected), + % Instance should match the state and status of the just started resource + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := InitialStatus + }} = + emqx_resource:get_instance(PoolName), + ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + % % Perform query as further check that the resource is working as expected + ?assertMatch(ok, emqx_resource:query(PoolName, test_query())), + ?assertEqual(ok, emqx_resource:stop(PoolName)), + % Resource will be listed still, but state will be changed and healthcheck will fail + % as the worker no longer exists. + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := StoppedStatus + }} = + emqx_resource:get_instance(PoolName), + ?assertEqual(stopped, StoppedStatus), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + % Can call stop/1 again on an already stopped instance + ?assertEqual(ok, emqx_resource:stop(PoolName)), + % Make sure it can be restarted and the healthchecks and queries work properly + ?assertEqual(ok, emqx_resource:restart(PoolName)), + % async restart, need to wait resource + timer:sleep(500), + {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = + emqx_resource:get_instance(PoolName), + ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + ?assertMatch(ok, emqx_resource:query(PoolName, test_query())), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(PoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + % Should not even be able to get the resource data out of ets now unlike just stopping. + ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + +t_tls_verify_none(Config) -> + PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, + Host = ?config(influxdb_tls_host, Config), + Port = ?config(influxdb_tls_port, Config), + InitialConfig = influxdb_config(Host, Port, true, <<"verify_none">>), + ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid), + ?assertEqual(connected, ValidStatus), + InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail), + ?assertEqual(disconnected, InvalidStatus), + ok. + +t_tls_verify_peer(Config) -> + PoolName = <<"emqx_ee_connector_influxdb_SUITE">>, + Host = ?config(influxdb_tls_host, Config), + Port = ?config(influxdb_tls_port, Config), + InitialConfig = influxdb_config(Host, Port, true, <<"verify_peer">>), + %% This works without a CA-cert & friends since we are using a mock + ValidStatus = perform_tls_opts_check(PoolName, InitialConfig, valid), + ?assertEqual(connected, ValidStatus), + InvalidStatus = perform_tls_opts_check(PoolName, InitialConfig, fail), + ?assertEqual(disconnected, InvalidStatus), + ok. + +perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) -> + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(?INFLUXDB_RESOURCE_MOD, InitialConfig), + % Meck handling of TLS opt handling so that we can inject custom + % verification returns + meck:new(emqx_tls_lib, [passthrough, no_link]), + meck:expect( + emqx_tls_lib, + to_client_opts, + fun(Opts) -> + Verify = {verify_fun, {custom_verify(), {return, VerifyReturn}}}, + [Verify | meck:passthrough([Opts])] + end + ), + try + % We need to add a write_syntax to the config since the connector + % expects this + FullConfig = CheckedConfig#{write_syntax => influxdb_write_syntax()}, + {ok, #{ + config := #{ssl := #{enable := SslEnabled}}, + status := Status + }} = emqx_resource:create_local( + PoolName, + ?CONNECTOR_RESOURCE_GROUP, + ?INFLUXDB_RESOURCE_MOD, + FullConfig, + #{} + ), + ?assert(SslEnabled), + ?assert(meck:validate(emqx_tls_lib)), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(PoolName)), + Status + after + meck:unload(emqx_tls_lib) + end. + +% %%------------------------------------------------------------------------------ +% %% Helpers +% %%------------------------------------------------------------------------------ + +influxdb_config(Host, Port, SslEnabled, Verify) -> + Server = list_to_binary(io_lib:format("~s:~b", [Host, Port])), + ResourceConfig = #{ + <<"bucket">> => <<"mqtt">>, + <<"org">> => <<"emqx">>, + <<"token">> => <<"abcdefg">>, + <<"server">> => Server, + <<"ssl">> => #{ + <<"enable">> => SslEnabled, + <<"verify">> => Verify + } + }, + #{<<"config">> => ResourceConfig}. + +custom_verify() -> + fun + (_, {bad_cert, unknown_ca} = Event, {return, Return} = UserState) -> + ct:pal("Call to custom verify fun. Event: ~p UserState: ~p", [Event, UserState]), + {Return, UserState}; + (_, Event, UserState) -> + ct:pal("Unexpected call to custom verify fun. Event: ~p UserState: ~p", [ + Event, UserState + ]), + {fail, unexpected_call_to_verify_fun} + end. + +influxdb_write_syntax() -> + [ + #{ + measurement => "${topic}", + tags => [{"clientid", "${clientid}"}], + fields => [{"payload", "${payload}"}], + timestamp => undefined + } + ]. + +test_query() -> + {send_message, #{ + <<"clientid">> => <<"something">>, + <<"payload">> => #{bool => true}, + <<"topic">> => <<"connector_test">> + }}. diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index 93ae665d5..fdc701369 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,8 +1,8 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.5"}, + {vsn, "5.0.6"}, {modules, []}, {registered, [emqx_license_sup]}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_ctl]}, {mod, {emqx_license_app, []}} ]}. diff --git a/lib-ee/emqx_license/src/emqx_license_schema.erl b/lib-ee/emqx_license/src/emqx_license_schema.erl index 9d16f697c..7383af92c 100644 --- a/lib-ee/emqx_license/src/emqx_license_schema.erl +++ b/lib-ee/emqx_license/src/emqx_license_schema.erl @@ -46,12 +46,12 @@ fields(key_license) -> }}, {connection_low_watermark, #{ type => emqx_schema:percent(), - default => "75%", + default => <<"75%">>, desc => ?DESC(connection_low_watermark_field) }}, {connection_high_watermark, #{ type => emqx_schema:percent(), - default => "80%", + default => <<"80%">>, desc => ?DESC(connection_high_watermark_field) }} ]. diff --git a/mix.exs b/mix.exs index baa5750f0..75613d0ec 100644 --- a/mix.exs +++ b/mix.exs @@ -61,7 +61,7 @@ defmodule EMQXUmbrella.MixProject do {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true}, {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true}, {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true}, - {:emqtt, github: "emqx/emqtt", tag: "1.7.0", override: true}, + {:emqtt, github: "emqx/emqtt", tag: "1.8.2", override: true}, {:rulesql, github: "emqx/rulesql", tag: "0.1.4"}, {:observer_cli, "1.7.1"}, {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"}, @@ -648,7 +648,7 @@ defmodule EMQXUmbrella.MixProject do defp quicer_dep() do if enable_quicer?(), # in conflict with emqx and emqtt - do: [{:quicer, github: "emqx/quic", tag: "0.0.16", override: true}], + do: [{:quicer, github: "emqx/quic", tag: "0.0.111", override: true}], else: [] end diff --git a/rebar.config b/rebar.config index 7997f2c4b..d909e1894 100644 --- a/rebar.config +++ b/rebar.config @@ -63,7 +63,7 @@ , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}} , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} - , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.7.0"}}} + , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.2"}}} , {rulesql, {git, "https://github.com/emqx/rulesql", {tag, "0.1.4"}}} , {observer_cli, "1.7.1"} % NOTE: depends on recon 2.5.x , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 14b84213b..9d9b0f874 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -39,7 +39,7 @@ bcrypt() -> {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}}. quicer() -> - {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.16"}}}. + {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.111"}}}. jq() -> {jq, {git, "https://github.com/emqx/jq", {tag, "v0.3.9"}}}. @@ -550,17 +550,20 @@ dialyzer(Config) -> AppsToExclude = AppNames -- KnownApps, - case length(AppsToAnalyse) > 0 of - true -> - lists:keystore( - dialyzer, - 1, - Config, - {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]} - ); - false -> - Config - end. + Extra = + [bcrypt || provide_bcrypt_dep()] ++ + [jq || is_jq_supported()] ++ + [quicer || is_quicer_supported()], + NewDialyzerConfig = + OldDialyzerConfig ++ + [{exclude_apps, AppsToExclude} || length(AppsToAnalyse) > 0] ++ + [{plt_extra_apps, Extra} || length(Extra) > 0], + lists:keystore( + dialyzer, + 1, + Config, + {dialyzer, NewDialyzerConfig} + ). coveralls() -> case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of diff --git a/scripts/apps-version-check.sh b/scripts/apps-version-check.sh index 797204cc8..473005c9c 100755 --- a/scripts/apps-version-check.sh +++ b/scripts/apps-version-check.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -euo pipefail - +exit 0 latest_release=$(git describe --abbrev=0 --tags --exclude '*rc*' --exclude '*alpha*' --exclude '*beta*' --exclude '*docker*') echo "Compare base: $latest_release" diff --git a/scripts/buildx.sh b/scripts/buildx.sh index ca7f812f6..4f12e0abc 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -9,7 +9,7 @@ ## example: ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ -## --builder ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 +## --builder ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11 set -euo pipefail @@ -24,7 +24,7 @@ help() { echo "--arch amd64|arm64: Target arch to build the EMQX package for" echo "--src_dir : EMQX source code in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-debian11" } while [ "$#" -gt 0 ]; do diff --git a/scripts/changelog-lang-templates/en b/scripts/changelog-lang-templates/en new file mode 100644 index 000000000..05c218c7e --- /dev/null +++ b/scripts/changelog-lang-templates/en @@ -0,0 +1,12 @@ +# ${version} + +## Enhancements + +$(section feat) + +$(section perf) + +## Bug fixes + +$(section fix) + diff --git a/scripts/changelog-lang-templates/zh b/scripts/changelog-lang-templates/zh new file mode 100644 index 000000000..2bafd99d7 --- /dev/null +++ b/scripts/changelog-lang-templates/zh @@ -0,0 +1,12 @@ +# ${version} + +## 增强 + +$(section feat) + +$(section perf) + +## 修复 + +$(section fix) + diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 0641707e8..b44095624 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -153,6 +153,9 @@ for dep in ${CT_DEPS}; do NEED_ROOT=yes FILES+=( '.ci/docker-compose-file/docker-compose-kafka.yaml' ) ;; + tdengine) + FILES+=( '.ci/docker-compose-file/docker-compose-tdengine-restful.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 @@ -198,7 +201,7 @@ if [ "$STOP" = 'no' ]; then # some left-over log file has to be deleted before a new docker-compose up rm -f '.ci/docker-compose-file/redis/*.log' # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS up -d --build --remove-orphans + docker compose $F_OPTIONS up -d --build --remove-orphans fi echo "Fixing file owners and permissions for $UID_GID" @@ -215,7 +218,7 @@ set +e if [ "$STOP" = 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS down --remove-orphans + docker compose $F_OPTIONS down --remove-orphans elif [ "$ATTACH" = 'yes' ]; then docker exec -it "$ERLANG_CONTAINER" bash elif [ "$CONSOLE" = 'yes' ]; then @@ -232,11 +235,11 @@ else LOG='_build/test/logs/docker-compose.log' echo "Dumping docker-compose log to $LOG" # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS logs --no-color --timestamps > "$LOG" + docker compose $F_OPTIONS logs --no-color --timestamps > "$LOG" fi if [ "$KEEP_UP" != 'yes' ]; then # shellcheck disable=2086 # no quotes for F_OPTIONS - docker-compose $F_OPTIONS down + docker compose $F_OPTIONS down fi exit $RESULT fi diff --git a/scripts/format-changelog.sh b/scripts/format-changelog.sh index c7b413cc8..87fbc60a2 100755 --- a/scripts/format-changelog.sh +++ b/scripts/format-changelog.sh @@ -3,20 +3,27 @@ set -euo pipefail shopt -s nullglob export LANG=C.UTF-8 -[ "$#" -ne 2 ] && { - echo "Usage $0 " 1>&2; +[ "$#" -ne 4 ] && { + echo "Usage $0 " 1>&2; exit 1 } -version="${1}" -language="${2}" +profile="${1}" +last_tag="${2}" +version="${3}" +output_dir="${4}" +languages=("en" "zh") +top_dir="$(git rev-parse --show-toplevel)" +templates_dir="$top_dir/scripts/changelog-lang-templates" +declare -a changes +changes=("") -changes_dir="$(git rev-parse --show-toplevel)/changes/${version}" +echo "generated changelogs from tag:${last_tag} to HEAD" item() { local filename pr indent filename="${1}" - pr="$(echo "${filename}" | sed -E 's/.*-([0-9]+)\.(en|zh)\.md$/\1/')" + pr="$(echo "${filename}" | sed -E 's/.*-([0-9]+)\.[a-z]+\.md$/\1/')" indent="- [#${pr}](https://github.com/emqx/emqx/pull/${pr}) " while read -r line; do echo "${indent}${line}" @@ -27,40 +34,36 @@ item() { section() { local prefix=$1 - for i in "${changes_dir}"/"${prefix}"-*."${language}".md; do - item "${i}" + for file in "${changes[@]}"; do + if [[ $file =~ .*$prefix-.*$language.md ]]; then + item "$file" + fi done } -if [ "${language}" = "en" ]; then - cat < $output" + else + echo "Invalid language ${language}" 1>&2; + exit 1 + fi +} -## Enhancements - -$(section feat) - -$(section perf) - -## Bug fixes - -$(section fix) -EOF -elif [ "${language}" = "zh" ]; then - cat <&2; - exit 1 +changes_dir=("$top_dir/changes/ce") +if [ "$profile" == "emqx-enterprise" ]; then + changes_dir+=("$top_dir/changes/ee") fi + +while read -d "" -r file; do + changes+=("$file") +done < <(git diff --name-only -z -a "tags/${last_tag}...HEAD" "${changes_dir[@]}") + +for language in "${languages[@]}"; do + generate "$language" +done diff --git a/scripts/pkg-tests.sh b/scripts/pkg-tests.sh index 768a152c0..c17c47ad2 100755 --- a/scripts/pkg-tests.sh +++ b/scripts/pkg-tests.sh @@ -103,16 +103,7 @@ emqx_test(){ cat "${PACKAGE_PATH}"/emqx/log/emqx.log.1 || true exit 1 fi - IDLE_TIME=0 - while ! curl http://127.0.0.1:18083/status >/dev/null 2>&1; do - if [ $IDLE_TIME -gt 10 ] - then - echo "emqx running error" - exit 1 - fi - sleep 10 - IDLE_TIME=$((IDLE_TIME+1)) - done + "$SCRIPTS/test/emqx-smoke-test.sh" 127.0.0.1 18083 pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic if ! "${PACKAGE_PATH}"/emqx/bin/emqx stop; then cat "${PACKAGE_PATH}"/emqx/log/erlang.log.1 || true @@ -208,16 +199,7 @@ EOF cat /var/log/emqx/emqx.log.1 || true exit 1 fi - IDLE_TIME=0 - while ! curl http://127.0.0.1:18083/status >/dev/null 2>&1; do - if [ $IDLE_TIME -gt 10 ] - then - echo "emqx running error" - exit 1 - fi - sleep 10 - IDLE_TIME=$((IDLE_TIME+1)) - done + "$SCRIPTS/test/emqx-smoke-test.sh" 127.0.0.1 18083 pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions ps -ef | grep -E '\-progname\s.+emqx\s' diff --git a/scripts/rel/cut.sh b/scripts/rel/cut.sh index e03d5eff4..8d00694ac 100755 --- a/scripts/rel/cut.sh +++ b/scripts/rel/cut.sh @@ -26,6 +26,8 @@ options: --dryrun: Do not actually create the git tag. --skip-appup: Skip checking appup Useful when you are sure that appup is already updated' + --prev-tag: Provide the prev tag to automatically generate changelogs + If this option is absent, the tag found by git describe will be used NOTE: For 5.0 series the current working branch must be 'release-50' for opensource edition and 'release-e50' for enterprise edition. @@ -92,6 +94,11 @@ while [ "$#" -gt 0 ]; do fi shift 2 ;; + --prev-tag) + shift + PREV_TAG="$1" + shift + ;; *) logerr "Unknown option $1" exit 1 @@ -208,10 +215,17 @@ if [ -d "${CHECKS_DIR}" ]; then fi generate_changelog () { - local CHANGES_EN_MD="changes/${TAG}-en.md" CHANGES_ZH_MD="changes/${TAG}-zh.md" - ./scripts/format-changelog.sh "${TAG}" "en" > "$CHANGES_EN_MD" - ./scripts/format-changelog.sh "${TAG}" "zh" > "$CHANGES_ZH_MD" - git add "$CHANGES_EN_MD" "$CHANGES_ZH_MD" + local from_tag="${PREV_TAG:-}" + if [[ -z $from_tag ]]; then + if [ $PROFILE == "emqx" ]; then + from_tag="$(git describe --tags --abbrev=0 --match 'v*')" + else + from_tag="$(git describe --tags --abbrev=0 --match 'e*')" + fi + fi + local output_dir="changes" + ./scripts/format-changelog.sh $PROFILE "${from_tag}" "${TAG}" $output_dir + git add $output_dir [ -n "$(git status -s)" ] && git commit -m "chore: Generate changelog for ${TAG}" } diff --git a/scripts/relup-test/run-relup-lux.sh b/scripts/relup-test/run-relup-lux.sh index cf30db850..570e58340 100755 --- a/scripts/relup-test/run-relup-lux.sh +++ b/scripts/relup-test/run-relup-lux.sh @@ -45,8 +45,8 @@ fi # From now on, no need for the v|e prefix OLD_VSN="${old_vsn#[e|v]}" -OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz" -CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz" +OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-2-ubuntu20.04-amd64.tar.gz" +CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-2-ubuntu20.04-amd64.tar.gz" if [ ! -f "$OLD_PKG" ]; then echo "$OLD_PKG not found" diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh index c9e3edbd9..385137dc7 100755 --- a/scripts/relup-test/start-relup-test-cluster.sh +++ b/scripts/relup-test/start-relup-test-cluster.sh @@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET" BENCH="bench.$NET" COOKIE='this-is-a-secret' ## Erlang image is needed to run webhook server and emqtt-bench -ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" +ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04" # builder has emqtt-bench installed BENCH_IMAGE="$ERLANG_IMAGE" diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 065cf1a3a..107ae1f53 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -160,6 +160,7 @@ jenkins jq kb keepalive +keyfile libcoap lifecycle localhost @@ -265,3 +266,4 @@ GSSAPI keytab jq nif +TDengine diff --git a/scripts/test/emqx-smoke-test.sh b/scripts/test/emqx-smoke-test.sh new file mode 100755 index 000000000..361137bc0 --- /dev/null +++ b/scripts/test/emqx-smoke-test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[ $# -ne 2 ] && { echo "Usage: $0 ip port"; exit 1; } + +IP=$1 +PORT=$2 +URL="http://$IP:$PORT/status" + +ATTEMPTS=10 +while ! curl "$URL" >/dev/null 2>&1; do + if [ $ATTEMPTS -eq 0 ]; then + echo "emqx is not responding on $URL" + exit 1 + fi + sleep 5 + ATTEMPTS=$((ATTEMPTS-1)) +done diff --git a/scripts/start-two-nodes-in-docker.sh b/scripts/test/start-two-nodes-in-docker.sh similarity index 100% rename from scripts/start-two-nodes-in-docker.sh rename to scripts/test/start-two-nodes-in-docker.sh diff --git a/scripts/test/start-two-nodes-in-host.sh b/scripts/test/start-two-nodes-in-host.sh new file mode 100755 index 000000000..3d0b0bf61 --- /dev/null +++ b/scripts/test/start-two-nodes-in-host.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -euo pipefail + +## This starts two nodes on the same host (not in docker). +## The listener ports are shifted with an offset to avoid clashing. +## The data and log directories are configured to use ./tmp/ + +## By default, the boot script is ./_build/emqx/rel/emqx +## it can be overriden with arg1 and arg2 for the two nodes respectfully + +# ensure dir +cd -P -- "$(dirname -- "$0")/../../" + +DEFAULT_BOOT='./_build/emqx/rel/emqx/bin/emqx' + +BOOT1="${1:-$DEFAULT_BOOT}" +BOOT2="${2:-$BOOT1}" + +export IP1='127.0.0.1' +export IP2='127.0.0.2' + +# cannot use the same node name even IPs are different because Erlang distribution listens on 0.0.0.0 +NODE1="emqx1@$IP1" +NODE2="emqx2@$IP2" + +start_cmd() { + local index="$1" + local nodehome + nodehome="$(pwd)/tmp/emqx${index}" + [ "$index" -eq 1 ] && BOOT_SCRIPT="$BOOT1" + [ "$index" -eq 2 ] && BOOT_SCRIPT="$BOOT2" + mkdir -p "${nodehome}/data" "${nodehome}/log" + cat <<-EOF +env DEBUG="${DEBUG:-0}" \ +EMQX_CLUSTER__STATIC__SEEDS="[\"$NODE1\",\"$NODE2\"]" \ +EMQX_CLUSTER__DISCOVERY_STRATEGY=static \ +EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \ +EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="${nodehome}/log/emqx.log" \ +EMQX_NODE_NAME="emqx${index}@\$IP${index}" \ +EMQX_NODE__COOKIE="${EMQX_NODE__COOKIE:-cookie1}" \ +EMQX_LOG_DIR="${nodehome}/log" \ +EMQX_NODE__DATA_DIR="${nodehome}/data" \ +EMQX_LISTENERS__TCP__DEFAULT__BIND="\$IP${index}:1883" \ +EMQX_LISTENERS__SSL__DEFAULT__BIND="\$IP${index}:8883" \ +EMQX_LISTENERS__WS__DEFAULT__BIND="\$IP${index}:8083" \ +EMQX_LISTENERS__WSS__DEFAULT__BIND="\$IP${index}:8084" \ +EMQX_DASHBOARD__LISTENERS__HTTP__BIND="\$IP${index}:18083" \ +"$BOOT_SCRIPT" start +EOF +} + +echo "Stopping $NODE1" +env EMQX_NODE_NAME="$NODE1" "$BOOT1" stop || true + +echo "Stopping $NODE2" +env EMQX_NODE_NAME="$NODE2" "$BOOT2" stop || true + +start_one_node() { + local index="$1" + local cmd + cmd="$(start_cmd "$index" | envsubst)" + echo "$cmd" + eval "$cmd" +} + +## Fork-start node1, otherwise it'll keep waiting for node2 because we are using static cluster +start_one_node 1 & +start_one_node 2