Add custom tag data parser (#132)

* Add formatString convenience function

* Use String& for wsLog, wsErr and wsSerial

* Add tag data parser and parse tag data

* Make logLine use String&

* Fix issue with formatString

* Reuse payloadLength in processTagReturnData

* Fix parsing of unsigned/signed inetegers and cleanup

* Use c++17 standard

* Cleanup logging
This commit is contained in:
Moritz Wirger
2023-09-26 22:51:57 +02:00
committed by GitHub
parent 4d08454fff
commit 2e44889b19
10 changed files with 363 additions and 31 deletions

View File

@@ -12,5 +12,5 @@
void initTime(void* parameter);
void logLine(const char* buffer);
void logLine(String text);
void logLine(const String& text);
void logStartUp();

View File

@@ -0,0 +1,140 @@
/// @file tagdata.h
/// @author Moritz Wirger (contact@wirmo.de)
/// @brief Custom tag data parser and helpers
#pragma once
#include <Arduino.h>
#include <ArduinoJson.h>
#include <optional>
#include <unordered_map>
#include <vector>
#include "storage.h"
#include "system.h"
#include "web.h"
/// @brief Functions for custom tag data parser
namespace TagData {
/// @brief All available data types
enum class Type {
/// @brief Signed integer type
INT,
/// @brief Unsigned integer type
UINT,
/// @brief Float type
FLOAT,
/// @brief String type
STRING,
/// @brief Not a type, just a helper to determine max type
MAX,
};
/// @brief Field that can be parsed
struct Field {
/// @brief Field name
String name;
/// @brief Field type
Type type;
/// @brief Field byte length
uint8_t length;
/// @brief Number of decimals numeric types
uint8_t decimals;
/// @brief Optional multiplication
std::optional<double> mult;
Field(const String &name, const Type type, const uint8_t length, uint8_t decimals = 0, std::optional<double> mult = std::nullopt)
: name(name), type(type), length(length), decimals(decimals), mult(mult) {}
};
/// @brief Parser for parsing custom tag data
struct Parser {
/// @brief Parser name
String name;
/// @brief Parsed fields
std::vector<Field> fields = {};
};
/// @brief Maps parser id to parser
extern std::unordered_map<size_t, Parser> parsers;
/// @brief Load all parsers from the given json file
/// @param filename File name
extern void loadParsers(const String &filename);
/// @brief Parse the incoming custom message
/// @param src Source mac address
/// @param id Message identifier
/// @param data Payload
/// @param len Payload length
extern void parse(const uint8_t src[8], const size_t id, const uint8_t *data, const uint8_t len);
/// @brief Convert the given byte array @ref data with given @ref length to an unsigned integer
///
/// Will also convert non standard integer sizes (e.g. 3, 5, 6, and 7 bytes)
/// @tparam T Unsigned integer type
/// @param data Byte array
/// @param length Length of byte array
/// @return Unsigned integer
template <typename T, std::enable_if_t<std::is_unsigned_v<T> && std::is_integral_v<T>, bool> = true>
inline T bytesTo(const uint8_t *data, const uint8_t length) {
T value = 0;
for (int i = 0; i < length; i++) {
value |= (data[i] & 0xFF) << (8 * i);
}
return value;
}
/// @brief Convert the given byte array @ref data with given @ref length to a signed integer
///
/// Will also convert non standard integer sizes (e.g. 3, 5, 6, and 7 bytes)
/// @tparam T Signed integer type
/// @param data Byte array
/// @param length Length of byte array
/// @return Signed integer
template <typename T, std::enable_if_t<std::is_signed_v<T> && std::is_integral_v<T>, bool> = true>
inline T bytesTo(const uint8_t *data, const uint8_t length) {
T value = 0;
for (int i = 0; i < length; ++i) {
value |= (data[i] & 0xFF) << (8 * i);
}
// If data is smaller than T and last byte is negative set all upper bytes negative
if (length < sizeof(T) && (data[length - 1] & 0x80) != 0) {
value |= ~((1 << (length * 8)) - 1);
}
return value;
}
/// @brief Convert the given byte array to a float/double
/// @param data Byte array, should be at least 4/8 bytes long
/// @param length Length of byte array
/// @return float/double
template <typename T, std::enable_if_t<std::is_floating_point_v<T>, bool> = true>
inline T bytesTo(const uint8_t *data, const uint8_t length) {
const size_t len = sizeof(T) < length ? sizeof(T) : length;
T value;
memcpy(&value, data, len);
return value;
}
/// @brief Convert the given byte array to a string
/// @param data Byte array representing a string
/// @param length Length of byte array
/// @return String
template <typename T, std::enable_if_t<std::is_same_v<T, String>, bool> = true>
inline T bytesTo(const uint8_t *data, int length) {
return T(data, length);
}
/// @brief Convert the given byte array to a string
/// @param data Byte array representing a string
/// @param length Length of byte array
/// @return std::string
template <typename T, std::enable_if_t<std::is_same_v<T, std::string>, bool> = true>
inline T bytesTo(const uint8_t *data, int length) {
return T(data, data + length);
}
} // namespace TagData

View File

@@ -100,7 +100,7 @@ static bool httpGetJson(String &url, JsonDocument &json, const uint16_t timeout,
///
/// @param str String to check
/// @return True if empty or null, false if not
static inline bool isEmptyOrNull(const String &str) {
inline bool isEmptyOrNull(const String &str) {
return str.isEmpty() || str == "null";
}
@@ -145,4 +145,19 @@ class Timer {
unsigned long previousMillis_;
};
/// @brief Create a String from format
/// @param buffer Buffer to use for sprintf
/// @param format String format
/// @return String
template <size_t bufSize>
inline String formatString(char buffer[bufSize], const char *format, ...) {
va_list args;
va_start(args, format);
const size_t size = vsnprintf(buffer, bufSize, format, args);
va_end(args);
return String(buffer, size);
}
} // namespace util

View File

@@ -6,12 +6,12 @@
void init_web();
void doImageUpload(AsyncWebServerRequest *request, String filename, size_t index, uint8_t *data, size_t len, bool final);
void doJsonUpload(AsyncWebServerRequest *request);
void wsLog(String text);
void wsErr(String text);
void wsLog(const String &text);
void wsErr(const String &text);
void wsSendTaginfo(const uint8_t *mac, uint8_t syncMode);
void wsSendSysteminfo();
void wsSendAPitem(struct APlist *apitem);
void wsSerial(String text);
void wsSerial(const String &text);
uint8_t wsClientCount();
extern AsyncWebSocket ws;

View File

@@ -26,7 +26,10 @@ board_build.filesystem = littlefs
monitor_filters = esp32_exception_decoder
monitor_speed = 115200
board_build.f_cpu = 240000000L
build_unflags =
-std=gnu++11
build_flags =
-std=gnu++17
-D BUILD_ENV_NAME=$PIOENV
-D BUILD_TIME=$UNIX_TIME
-D USER_SETUP_LOADED
@@ -42,8 +45,10 @@ platform = https://github.com/platformio/platform-espressif32.git
board=lolin_s2_mini
board_build.partitions = default.csv
build_unflags =
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
-std=gnu++11
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
build_flags =
-std=gnu++17
${env.build_flags}
-D OPENEPAPERLINK_MINI_AP_PCB
-D ARDUINO_USB_MODE=0
@@ -64,7 +69,7 @@ build_flags =
-D FLASHER_LED=15
-D FLASHER_RGB_LED=33
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
board_build.psram_type=qspi_opi
board_upload.maximum_size = 4194304
board_upload.maximum_ram_size = 327680
@@ -79,8 +84,10 @@ platform = https://github.com/platformio/platform-espressif32.git
board=lolin_s2_mini
board_build.partitions = default.csv
build_unflags =
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
-std=gnu++11
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
build_flags =
-std=gnu++17
${env.build_flags}
-D OPENEPAPERLINK_NANO_AP_PCB
-D ARDUINO_USB_MODE=0
@@ -99,7 +106,7 @@ build_flags =
-D FLASHER_LED=15
-D FLASHER_RGB_LED=-1
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
board_build.psram_type=qspi_opi
board_upload.maximum_size = 4194304
board_upload.maximum_ram_size = 327680
@@ -114,9 +121,11 @@ platform = https://github.com/platformio/platform-espressif32.git
board = esp32-s3-devkitc-1
board_build.partitions = default_16MB.csv
build_unflags =
-D ARDUINO_USB_MODE=1
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
-std=gnu++11
-D ARDUINO_USB_MODE=1
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
build_flags =
-std=gnu++17
${env.build_flags}
-D OPENEPAPERLINK_PCB
-D ARDUINO_USB_MODE=0
@@ -158,7 +167,7 @@ build_flags =
-D FLASHER_LED=21
-D FLASHER_RGB_LED=48
build_src_filter =
+<*>-<espflasher.cpp>
+<*>-<espflasher.cpp>
board_build.flash_mode=qio
board_build.arduino.memory_type = qio_opi
board_build.psram_type=qspi_opi
@@ -173,7 +182,10 @@ board_upload.flash_size = 16MB
[env:Simple_AP]
board = esp32dev
board_build.partitions = default.csv
build_unflags =
-std=gnu++11
build_flags =
-std=gnu++17
${env.build_flags}
-D CORE_DEBUG_LEVEL=0
-D SIMPLE_AP
@@ -188,7 +200,7 @@ build_flags =
-D FLASHER_AP_RXD=16
-D FLASHER_LED=22
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
; ----------------------------------------------------------------------------------------
; !!! this configuration expects an wemos_d1_mini32
@@ -197,7 +209,10 @@ build_src_filter =
[env:Wemos_d1_mini32_AP]
board = wemos_d1_mini32
board_build.partitions = default.csv
build_unflags =
-std=gnu++11
build_flags =
-std=gnu++17
${env.build_flags}
-D CORE_DEBUG_LEVEL=0
@@ -214,7 +229,7 @@ build_flags =
-D FLASHER_AP_RXD=17
-D FLASHER_LED=22
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
; ----------------------------------------------------------------------------------------
; !!! this configuration expects an m5stack esp32
@@ -224,7 +239,10 @@ build_src_filter =
platform = espressif32
board = m5stack-core-esp32
board_build.partitions = esp32_sdcard.csv
build_unflags =
-std=gnu++11
build_flags =
-std=gnu++17
${env.build_flags}
-D CORE_DEBUG_LEVEL=0
@@ -251,7 +269,7 @@ build_flags =
-D ILI9341_DRIVER
-D SMOOTH_FONT
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
; ----------------------------------------------------------------------------------------
; !!! this configuration expects an ESP32-S3 16MB Flash 8MB RAM
;
@@ -260,12 +278,14 @@ build_src_filter =
board = esp32-s3-devkitc-1
board_build.partitions = large_spiffs_16MB.csv
build_unflags =
-std=gnu++11
-D ARDUINO_USB_MODE=1
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
-D ILI9341_DRIVER
lib_deps =
${env.lib_deps}
build_flags =
-std=gnu++17
${env.build_flags}
-D YELLOW_IPS_AP
-D CORE_DEBUG_LEVEL=0
@@ -307,7 +327,7 @@ build_flags =
-D SERIAL_FLASHER_BOOT_HOLD_TIME_MS=50
-D SERIAL_FLASHER_RESET_HOLD_TIME_MS=100
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>
board_build.flash_mode=qio
board_build.arduino.memory_type = qio_opi
board_build.psram_type=qspi_opi
@@ -321,7 +341,10 @@ board_upload.flash_size = 16MB
[env:Sonoff_zb_bridge_P_AP]
board = esp32dev
board_build.partitions = default.csv
build_unflags =
-std=gnu++11
build_flags =
-std=gnu++17
${env.build_flags}
-D CORE_DEBUG_LEVEL=0
@@ -340,7 +363,7 @@ build_flags =
-D FLASHER_AP_RXD=23
-D FLASHER_LED=2
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
board_build.psram_type=qspi_opi
board_upload.maximum_size = 4194304
board_upload.maximum_ram_size = 327680
@@ -354,8 +377,10 @@ platform = https://github.com/platformio/platform-espressif32.git
board=lolin_s2_mini
board_build.partitions = default.csv
build_unflags =
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
-std=gnu++11
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
build_flags =
-std=gnu++17
${env.build_flags}
-D OPENEPAPERLINK_MINI_AP_PCB
-D ARDUINO_USB_MODE=0
@@ -375,7 +400,7 @@ build_flags =
-D FLASHER_LED=2
-D FLASHER_RGB_LED=-1
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
board_build.psram_type=qspi_opi
board_upload.maximum_size = 4194304
board_upload.maximum_ram_size = 327680
@@ -389,9 +414,11 @@ board_upload.flash_size = 4MB
board = esp32-s3-devkitc-1
board_build.partitions = 32MB_partition table.csv
build_unflags =
-D ARDUINO_USB_MODE=1
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
-std=gnu++11
-D ARDUINO_USB_MODE=1
-D CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y
build_flags =
-std=gnu++17
${env.build_flags}
-D OutdoorAP
-D HAS_RGB_LED
@@ -414,7 +441,7 @@ build_flags =
-D FLASHER_LED=21
-D FLASHER_RGB_LED=38
build_src_filter =
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
+<*>-<usbflasher.cpp>-<swd.cpp>-<espflasher.cpp>
board_build.flash_mode=opi
board_build.arduino.memory_type = opi_opi
board_build.psram_type=qspi_opi

View File

@@ -10,6 +10,7 @@
#include "storage.h"
#include "system.h"
#include "tag_db.h"
#include "tagdata.h"
#include "wifimanager.h"
#ifdef HAS_USB
@@ -126,6 +127,7 @@ void setup() {
#ifdef HAS_RGB_LED
rgbIdle();
#endif
TagData::loadParsers("/parsers.json");
loadDB("/current/tagDB.json");
cleanupCurrent();
xTaskCreate(APTask, "AP Process", 6000, NULL, 2, NULL);

View File

@@ -12,6 +12,7 @@
#include "storage.h"
#include "system.h"
#include "tag_db.h"
#include "tagdata.h"
#include "udp.h"
#include "util.h"
#include "web.h"
@@ -527,16 +528,17 @@ void processTagReturnData(struct espTagReturnData* trd, uint8_t len, bool local)
if (!checkCRC(trd, len)) {
return;
}
char buffer[64];
const uint8_t payloadLength = trd->len - 11;
// Replace this stuff with something that handles the data coming from the tag. This is here for demo purposes!
char buffer[64];
sprintf(buffer, "<TRD %02X%02X%02X%02X%02X%02X%02X%02X\n", trd->src[7], trd->src[6], trd->src[5], trd->src[4], trd->src[3], trd->src[2], trd->src[1], trd->src[0]);
wsLog((String)buffer);
sprintf(buffer, "TRD Data: len=%d, type=%d, ver=0x%08X\n", trd->len - 11, trd->returnData.dataType, trd->returnData.dataVer);
sprintf(buffer, "TRD Data: len=%d, type=%d, ver=0x%08X\n", payloadLength, trd->returnData.dataType, trd->returnData.dataVer);
wsLog((String)buffer);
uint8_t actualPayloadLength = trd->len - 11;
uint8_t* actualPayload = (uint8_t*)calloc(actualPayloadLength, 1);
memcpy(actualPayload, trd->returnData.data, actualPayloadLength);
TagData::parse(trd->src, trd->returnData.dataType, trd->returnData.data, payloadLength);
}
void refreshAllPending() {

View File

@@ -35,7 +35,7 @@ void logLine(const char* buffer) {
logLine(String(buffer));
}
void logLine(String text) {
void logLine(const String& text) {
time_t now;
time(&now);

View File

@@ -0,0 +1,146 @@
#include "tagdata.h"
#include "tag_db.h"
#include "util.h"
std::unordered_map<size_t, TagData::Parser> TagData::parsers = {};
void TagData::loadParsers(const String& filename) {
Serial.println("Reading parsers from file");
const long start = millis();
Storage.begin();
fs::File file = contentFS->open(filename, "r");
if (!file) {
Serial.println("loadParsers: Failed to open file");
return;
}
if (file.find("[")) {
StaticJsonDocument<1000> doc;
bool parsing = true;
while (parsing) {
DeserializationError err = deserializeJson(doc, file);
if (!err) {
const JsonObject parserDoc = doc[0];
const auto& id = parserDoc["id"];
const auto& name = parserDoc["name"];
if (!id || !name) {
Serial.printf("Error: Parser must have name and id\n");
continue;
}
Parser parser;
parser.name = name.as<String>();
for (const auto& parserField : parserDoc["parser"].as<JsonArray>()) {
const uint8_t type = parserField["type"].as<uint8_t>();
if (type >= (uint8_t)Type::MAX) {
Serial.printf("Error: Type %d is not a valid tag data parser data type\n", type);
continue;
}
const auto& mult = parserField["mult"];
const uint8_t decimals = parserField["decimals"].as<uint8_t>();
if (mult) {
parser.fields.emplace_back(parserField["name"].as<String>(),
static_cast<Type>(type),
parserField["length"].as<uint8_t>(),
decimals,
std::make_optional(mult.as<double>()));
} else {
parser.fields.emplace_back(parserField["name"].as<String>(),
static_cast<Type>(type),
parserField["length"].as<uint8_t>(),
decimals);
}
}
parsers.emplace(id.as<uint8_t>(), parser);
} else {
Serial.print(F("deserializeJson() failed: "));
Serial.println(err.c_str());
parsing = false;
}
parsing = parsing && file.find(",");
}
}
file.close();
Serial.printf("Loaded %d parsers in %d ms\n", parsers.size(), millis() - start);
}
void TagData::parse(const uint8_t src[8], const size_t id, const uint8_t* data, const uint8_t len) {
char buffer[64];
const auto it = parsers.find(id);
if (it == parsers.end()) {
const String log = util::formatString<64>(buffer, "Error: No parser with id %d found(%d)", id, parsers.size());
wsErr(log);
Serial.println(log);
return;
}
const String mac = util::formatString<64>(buffer, "%02X%02X%02X%02X%02X%02X%02X%02X.", src[7], src[6], src[5], src[4], src[3], src[2], src[1], src[0]);
uint16_t offset = 0;
for (const Field& field : it->second.fields) {
const String& name = field.name;
const uint8_t length = field.length;
if (offset + length > len) {
const String log = util::formatString<64>(buffer, "Error: Not enough data for field %s", name.c_str());
wsErr(log);
Serial.println(log);
return;
}
const Type type = field.type;
const uint8_t* fieldData = data + offset;
offset += length;
String value = "";
switch (type) {
case Type::INT: {
const double mult = field.mult.value_or(1.0);
value = String(bytesTo<int64_t>(fieldData, length) * mult, (unsigned int)field.decimals);
} break;
case Type::UINT: {
const double mult = field.mult.value_or(1.0f);
value = String(bytesTo<uint64_t>(fieldData, length) * mult, (unsigned int)field.decimals);
} break;
case Type::FLOAT: {
const double mult = field.mult.value_or(1.0f);
if (length == 4) {
value = String(bytesTo<float>(fieldData, length) * mult, (unsigned int)field.decimals);
} else if (length == 8) {
value = String(bytesTo<double>(fieldData, length) * mult, (unsigned int)field.decimals);
} else {
const String log = "Error: Float can only be 4 or 8 bytes long";
wsErr(log);
Serial.println(log);
}
} break;
case Type::STRING: {
value = bytesTo<String>(fieldData, length);
} break;
default:
const String log = util::formatString<64>(buffer, "Error: Type %d not implemented", static_cast<uint8_t>(type));
wsErr(log);
Serial.println(log);
break;
}
if (value.isEmpty()) {
const String log = util::formatString<64>(buffer, "Error: Empty value for field %s", name.c_str());
wsErr(log);
Serial.println(log);
continue;
}
const std::string varName = (mac + name).c_str();
setVarDB(varName, value);
Serial.printf("Set %s to %s\n", varName.c_str(), value.c_str());
}
}

View File

@@ -35,7 +35,7 @@ WifiManager wm;
SemaphoreHandle_t wsMutex;
uint32_t lastssidscan = 0;
void wsLog(String text) {
void wsLog(const String &text) {
StaticJsonDocument<250> doc;
doc["logMsg"] = text;
if (wsMutex) xSemaphoreTake(wsMutex, portMAX_DELAY);
@@ -43,7 +43,7 @@ void wsLog(String text) {
if (wsMutex) xSemaphoreGive(wsMutex);
}
void wsErr(String text) {
void wsErr(const String &text) {
StaticJsonDocument<250> doc;
doc["errMsg"] = text;
if (wsMutex) xSemaphoreTake(wsMutex, portMAX_DELAY);
@@ -167,7 +167,7 @@ void wsSendAPitem(struct APlist *apitem) {
if (wsMutex) xSemaphoreGive(wsMutex);
}
void wsSerial(String text) {
void wsSerial(const String &text) {
StaticJsonDocument<250> doc;
doc["console"] = text;
Serial.println(text);