Merge branch 'feature/update_mqtt_component' into 'master'

feat(mqtt): update mqtt component and examples from idf

See merge request sdk/ESP8266_RTOS_SDK!1587

Author: donghengqaz

components/mqtt/CMakeLists.txt

@@ -5,5 +5,3 @@ idf_component_register(SRCS "esp-mqtt/mqtt_client.c"
                     INCLUDE_DIRS esp-mqtt/include
                     PRIV_INCLUDE_DIRS "esp-mqtt/lib/include"
                     REQUIRES lwip http_parser mbedtls tcp_transport)
-
-target_compile_definitions(${COMPONENT_LIB} PUBLIC -DMQTT_SUPPORTED_FEATURE_WS_SUBPROTOCOL -DMQTT_SUPPORTED_FEATURE_TRANSPORT_ERR_REPORTING)

components/mqtt/Kconfig

@@ -86,6 +86,13 @@ menu "ESP-MQTT Configurations"
             Default config employs API locks to protect internal structures. It is possible to disable
             these locks if the user code doesn't access MQTT API from multiple concurrent tasks
 
+    config MQTT_TASK_PRIORITY
+        int "MQTT task priority"
+        default 5
+        depends on MQTT_USE_CUSTOM_CONFIG
+        help
+            MQTT task priority. Higher number denotes higher priority.
+
     config MQTT_TASK_CORE_SELECTION_ENABLED
         bool "Enable MQTT task core selection"
         default false

components/mqtt/component.mk

@@ -2,5 +2,3 @@ COMPONENT_SUBMODULES += esp-mqtt
 COMPONENT_ADD_INCLUDEDIRS := esp-mqtt/include
 COMPONENT_SRCDIRS :=  esp-mqtt esp-mqtt/lib
 COMPONENT_PRIV_INCLUDEDIRS := esp-mqtt/lib/include
-#Due to RTOS version is lower than idf 4.0, so move some define in file mqtt_supported_features.h to here.
-CFLAGS += -DMQTT_SUPPORTED_FEATURE_WS_SUBPROTOCOL -DMQTT_SUPPORTED_FEATURE_TRANSPORT_ERR_REPORTING

components/mqtt/esp-mqtt

@@ -0,0 +1,2 @@
+idf_component_register(SRC_DIRS "."
+                    PRIV_REQUIRES unity test_utils mqtt nvs_flash app_update)

components/mqtt/test/CMakeLists.txt

@@ -0,0 +1,4 @@
+#
+#Component Makefile
+#
+COMPONENT_ADD_LDFLAGS = -Wl,--whole-archive -l$(COMPONENT_NAME) -Wl,--no-whole-archive

components/mqtt/test/component.mk

@@ -0,0 +1,70 @@
+#include "test_utils.h"
+#include "mqtt_client.h"
+#include "unity.h"
+#include <sys/time.h>
+#include "nvs_flash.h"
+#include "esp_ota_ops.h"
+
+static void test_leak_setup(const char * file, long line)
+{
+    uint8_t mac[6];
+    struct timeval te;
+    gettimeofday(&te, NULL); // get current time
+    esp_read_mac(mac, ESP_MAC_WIFI_STA);
+    printf("%s:%ld: time=%ld.%lds, mac:" MACSTR "\n", file, line, te.tv_sec, te.tv_usec, MAC2STR(mac));
+    unity_reset_leak_checks();
+}
+
+TEST_CASE("mqtt init with invalid url", "[mqtt][leaks=0]")
+{
+    test_leak_setup(__FILE__, __LINE__);
+    const esp_mqtt_client_config_t mqtt_cfg = {
+            .uri = "INVALID",
+    };
+    esp_mqtt_client_handle_t client = esp_mqtt_client_init(&mqtt_cfg);
+    TEST_ASSERT_EQUAL(NULL, client );
+}
+
+TEST_CASE("mqtt init and deinit", "[mqtt][leaks=0]")
+{
+    test_leak_setup(__FILE__, __LINE__);
+    const esp_mqtt_client_config_t mqtt_cfg = {
+            // no connection takes place, but the uri has to be valid for init() to succeed
+            .uri = "mqtts://localhost:8883",
+    };
+    esp_mqtt_client_handle_t client = esp_mqtt_client_init(&mqtt_cfg);
+    TEST_ASSERT_NOT_EQUAL(NULL, client );
+    esp_mqtt_client_destroy(client);
+}
+
+static const char* this_bin_addr(void)
+{
+    spi_flash_mmap_handle_t out_handle;
+    const void *binary_address;
+    const esp_partition_t* partition = esp_ota_get_running_partition();
+    esp_partition_mmap(partition, 0, partition->size, SPI_FLASH_MMAP_DATA, &binary_address, &out_handle);
+    return binary_address;
+}
+
+TEST_CASE("mqtt enqueue and destroy outbox", "[mqtt][leaks=0]")
+{
+    const char * bin_addr = this_bin_addr();
+    test_leak_setup(__FILE__, __LINE__);
+    const int messages = 20;
+    const int size = 2000;
+    const esp_mqtt_client_config_t mqtt_cfg = {
+            // no connection takes place, but the uri has to be valid for init() to succeed
+            .uri = "mqtts://localhost:8883",
+    };
+    esp_mqtt_client_handle_t client = esp_mqtt_client_init(&mqtt_cfg);
+    TEST_ASSERT_NOT_EQUAL(NULL, client );
+    int bytes_before = esp_get_free_heap_size();
+    for (int i=0; i<messages; ++i) {
+        esp_mqtt_client_publish(client, "test", bin_addr, size, 1, 0);
+    }
+    int bytes_after = esp_get_free_heap_size();
+    // check that outbox allocated all messages on heap
+    TEST_ASSERT_GREATER_OR_EQUAL(messages*size, bytes_before - bytes_after);
+
+    esp_mqtt_client_destroy(client);
+}

components/mqtt/test/test_mqtt.c

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 from builtins import str
 import re
-import os
 import sys
 import ssl
 import paho.mqtt.client as mqtt
@@ -11,19 +10,8 @@
 import string
 import random
 
-try:
-    import IDF
-except ImportError:
-    # this is a test case write with tiny-test-fw.
-    # to run test cases outside tiny-test-fw,
-    # we need to set environment variable `TEST_FW_PATH`,
-    # then get and insert `TEST_FW_PATH` to sys path before import FW module
-    test_fw_path = os.getenv("TEST_FW_PATH")
-    if test_fw_path and test_fw_path not in sys.path:
-        sys.path.insert(0, test_fw_path)
-    import IDF
-
-import DUT
+from tiny_test_fw import DUT
+import ttfw_idf
 
 
 event_client_connected = Event()
@@ -52,6 +40,8 @@ def mqtt_client_task(client):
 
 def get_host_port_from_dut(dut1, config_option):
     value = re.search(r'\:\/\/([^:]+)\:([0-9]+)', dut1.app.get_sdkconfig()[config_option])
+    if value is None:
+        return None, None
     return value.group(1), int(value.group(2))
 
 
@@ -123,7 +113,7 @@ def test_single_config(dut, transport, qos, repeat, published):
     event_stop_client.clear()
 
 
-@IDF.idf_example_test(env_tag="Example_WIFI")
+@ttfw_idf.idf_custom_test(env_tag="Example_WIFI")
 def test_weekend_mqtt_publish(env, extra_data):
     # Using broker url dictionary for different transport
     global broker_host
@@ -137,13 +127,7 @@ def test_weekend_mqtt_publish(env, extra_data):
       3. Test evaluates python client received correct qos0 message
       4. Test ESP32 client received correct qos0 message
     """
-    dut1 = env.get_dut("mqtt_publish", "examples/protocols/mqtt/publish_test")
-    # check and log bin size
-    binary_file = os.path.join(dut1.app.binary_path, "mqtt_publish.bin")
-    bin_size = os.path.getsize(binary_file)
-    IDF.log_performance("mqtt_publish_bin_size", "{}KB"
-                        .format(bin_size // 1024))
-    IDF.check_performance("mqtt_publish_size", bin_size // 1024)
+    dut1 = env.get_dut("mqtt_publish_connect_test", "tools/test_apps/protocols/mqtt/publish_connect_test")
     # Look for host:port in sdkconfig
     try:
         # python client subscribes to the topic to which esp client publishes and vice versa
@@ -158,13 +142,16 @@ def test_weekend_mqtt_publish(env, extra_data):
         raise
     dut1.start_app()
     try:
-        ip_address = dut1.expect(re.compile(r" sta ip: ([^,]+),"), timeout=30)
+        ip_address = dut1.expect(re.compile(r" IPv4 address: ([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)"), timeout=30)
         print("Connected to AP with IP: {}".format(ip_address))
     except DUT.ExpectTimeout:
         print('ENV_TEST_FAILURE: Cannot connect to AP')
         raise
     for qos in [0, 1, 2]:
         for transport in ["tcp", "ssl", "ws", "wss"]:
+            if broker_host[transport] is None:
+                print('Skipping transport: {}...'.format(transport))
+                continue
             # simple test with empty message
             test_single_config(dut1, transport, qos, 0, 5)
             # decide on broker what level of test will pass (local broker works the best)
@@ -188,4 +175,4 @@ def test_weekend_mqtt_publish(env, extra_data):
 
 
 if __name__ == '__main__':
-    test_weekend_mqtt_publish()
+    test_weekend_mqtt_publish(dut=ttfw_idf.ESP32QEMUDUT if sys.argv[1:] == ['qemu'] else ttfw_idf.ESP32DUT)

components/mqtt/weekend_test/mqtt_publish_test.py

@@ -0,0 +1,7 @@
+CaseConfig:
+- name: test_weekend_mqtt_publish
+  overwrite:
+    dut:
+      class: ESP32QEMUDUT
+      package: ttfw_idf
+