Add generator support for PlatformIO (#718)

Add rules for running nanopb generator from PlatformIO build.
Added example and test for a PlatformIO based build.

.github/workflows/platformio.yaml

@@ -0,0 +1,47 @@
+name: platformio
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  platformio-example:
+    name: Build and run PlatformIO example
+    runs-on: ubuntu-latest
+    steps:
+      - name: ⤵️ Check out code from GitHub
+        uses: actions/checkout@v2
+
+      - name: Installing dependencies for local act
+        if: ${{ env.ACT }}
+        run: |
+          sudo apt update
+
+      - name: Installing common dependencies
+        run: |
+          sudo apt install -y python3-pip python3-protobuf protobuf-compiler
+
+      - name: Install and setup PlatformIO
+        run: |
+          pip3 install -U platformio
+          export PATH=~/.local/bin:$PATH
+
+      - name: Build PlatformIO package
+        run: pio package pack
+
+      - name: Extract PlatformIO package to example dir
+        run: |
+          mkdir -p examples/platformio/lib/nanopb
+          tar -xzf Nanopb-*.tar.gz -C examples/platformio/lib/nanopb
+          ls -l examples/platformio/lib/nanopb
+
+      - name: 🚀 Build
+        run: |
+          cd examples/platformio
+          pio run
+
+      - name: Run test without options
+        run: examples/platformio/.pio/build/pio_without_options/program
+
+      - name: Run test with options
+        run: examples/platformio/.pio/build/pio_with_options/program

examples/platformio/.gitignore

@@ -0,0 +1,5 @@
+.pio/
+.idea/
+cmake-build-*/
+CMakeLists.txt
+CMakeListsPrivate.txt

examples/platformio/platformio.ini

@@ -0,0 +1,35 @@
+;
+; You can setup `nanopb_protos` `nanopb_options` vars to generate code from proto files
+;
+; Generator will use next folders:
+;
+;   `$BUILD_DIR/nanopb/generated-src` - `*.pb.h` and `*.pb.c` files
+;   `$BUILD_DIR/nanopb/md5` - MD5 files to track changes in source .proto/.options
+;
+; Compiled `.pb.o` files will be located under `$BUILD_DIR/nanopb/generated-build`
+;
+; Example:
+
+[env:pio_with_options]
+platform = native
+lib_deps = Nanopb
+
+src_filter =
+    +<pio_with_options.c>
+
+; All path are relative to the `$PROJECT_DIR`
+nanopb_protos =
+    +<proto/pio_with_options.proto>
+nanopb_options =
+    --error-on-unmatched
+
+[env:pio_without_options]
+platform = native
+lib_deps = Nanopb
+
+src_filter =
+    +<pio_without_options.c>
+
+; All path are relative to the `$PROJECT_DIR`
+nanopb_protos =
+    +<proto/pio_without_options.proto>

examples/platformio/proto/pio_with_options.options

@@ -0,0 +1 @@
+TestMessageWithOptions.str max_size:16

examples/platformio/proto/pio_with_options.proto

@@ -0,0 +1,5 @@
+syntax = "proto3";
+
+message TestMessageWithOptions {
+  string str = 1;
+}

examples/platformio/proto/pio_without_options.proto

@@ -0,0 +1,5 @@
+syntax = "proto3";
+
+message TestMessageWithoutOptions {
+  int32 number = 1;
+}

examples/platformio/src/pio_with_options.c

@@ -0,0 +1,35 @@
+#include "pb_encode.h"
+#include "pb_decode.h"
+
+#include "test.h"
+
+#include "pio_with_options.pb.h"
+
+int main(int argc, char *argv[]) {
+
+    int status = 0;
+
+    uint8_t buffer[256];
+    pb_ostream_t ostream;
+    pb_istream_t istream;
+    size_t written;
+
+    TestMessageWithOptions original = TestMessageWithOptions_init_zero;
+    strcpy(original.str,"Hello");
+
+    ostream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+
+    TEST(pb_encode(&ostream, &TestMessageWithOptions_msg, &original));
+
+    written = ostream.bytes_written;
+
+    istream = pb_istream_from_buffer(buffer, written);
+
+    TestMessageWithOptions decoded = TestMessageWithOptions_init_zero;
+
+    TEST(pb_decode(&istream, &TestMessageWithOptions_msg, &decoded));
+
+    TEST(strcmp(decoded.str,"Hello") == 0);
+
+    return status;
+}

examples/platformio/src/pio_without_options.c

@@ -0,0 +1,35 @@
+#include "pb_encode.h"
+#include "pb_decode.h"
+
+#include "test.h"
+
+#include "pio_without_options.pb.h"
+
+int main(int argc, char *argv[]) {
+
+    int status = 0;
+
+    uint8_t buffer[256];
+    pb_ostream_t ostream;
+    pb_istream_t istream;
+    size_t written;
+
+    TestMessageWithoutOptions original = TestMessageWithoutOptions_init_zero;
+    original.number = 45;
+
+    ostream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+
+    TEST(pb_encode(&ostream, &TestMessageWithoutOptions_msg, &original));
+
+    written = ostream.bytes_written;
+
+    istream = pb_istream_from_buffer(buffer, written);
+
+    TestMessageWithoutOptions decoded = TestMessageWithoutOptions_init_zero;
+
+    TEST(pb_decode(&istream, &TestMessageWithoutOptions_msg, &decoded));
+
+    TEST(decoded.number == 45);
+
+    return status;
+}

examples/platformio/src/test.h

@@ -0,0 +1,9 @@
+#include <stdio.h>
+
+#define TEST(x) \
+    if (!(x)) { \
+        fprintf(stderr, "\033[31;1mFAILED:\033[22;39m %s:%d %s\n", __FILE__, __LINE__, #x); \
+        status = 1; \
+    } else { \
+        printf("\033[32;1mOK:\033[22;39m %s\n", #x); \
+    }

generator/platformio_generator.py

@@ -0,0 +1,127 @@
+import os
+import hashlib
+import pathlib
+from platformio import fs
+
+Import("env")
+
+nanopb_root = os.path.join(os.getcwd(), '..')
+
+project_dir = env.subst("$PROJECT_DIR")
+build_dir = env.subst("$BUILD_DIR")
+
+generated_src_dir = os.path.join(build_dir, 'nanopb', 'generated-src')
+generated_build_dir = os.path.join(build_dir, 'nanopb', 'generated-build')
+md5_dir = os.path.join(build_dir, 'nanopb', 'md5')
+
+nanopb_protos = env.GetProjectOption("nanopb_protos", "")
+nanopb_plugin_options = env.GetProjectOption("nanopb_options", "")
+
+if not nanopb_protos:
+    print("[nanopb] No `nanopb_protos` specified, exiting.")
+    exit(0)
+
+if isinstance(nanopb_plugin_options, (list, tuple)):
+    nanopb_plugin_options = " ".join(nanopb_plugin_options)
+
+nanopb_plugin_options = nanopb_plugin_options.split()
+
+protos_files = fs.match_src_files(project_dir, nanopb_protos)
+if not len(protos_files):
+    print("[nanopb] ERROR: No files matched pattern:")
+    print(f"nanopb_protos: {nanopb_protos}")
+    exit(1)
+
+protoc_generator = os.path.join(nanopb_root, 'generator', 'protoc')
+
+nanopb_options = ""
+nanopb_options += f" --nanopb_out={generated_src_dir}"
+for opt in nanopb_plugin_options:
+    nanopb_options += (" --nanopb_opt=" + opt)
+
+try:
+    os.makedirs(generated_src_dir)
+except FileExistsError:
+    pass
+
+try:
+    os.makedirs(md5_dir)
+except FileExistsError:
+    pass
+
+# Collect include dirs based on
+proto_include_dirs = set()
+for proto_file in protos_files:
+    proto_file_abs = os.path.join(project_dir, proto_file)
+    proto_dir = os.path.dirname(proto_file_abs)
+    proto_include_dirs.add(proto_dir)
+
+for proto_include_dir in proto_include_dirs:
+    nanopb_options += (" --proto_path=" + proto_include_dir)
+    nanopb_options += (" --nanopb_opt=-I" + proto_include_dir)
+
+for proto_file in protos_files:
+    proto_file_abs = os.path.join(project_dir, proto_file)
+
+    proto_file_path_abs = os.path.dirname(proto_file_abs)
+    proto_file_basename = os.path.basename(proto_file_abs)
+    proto_file_without_ext = os.path.splitext(proto_file_basename)[0]
+
+    proto_file_md5_abs = os.path.join(md5_dir, proto_file_basename + '.md5')
+    proto_file_current_md5 = hashlib.md5(pathlib.Path(proto_file_abs).read_bytes()).hexdigest()
+
+    options_file = proto_file_without_ext + ".options"
+    options_file_abs = os.path.join(proto_file_path_abs, options_file)
+    options_file_md5_abs = None
+    options_file_current_md5 = None
+    if pathlib.Path(options_file_abs).exists():
+        options_file_md5_abs = os.path.join(md5_dir, options_file + '.md5')
+        options_file_current_md5 = hashlib.md5(pathlib.Path(options_file_abs).read_bytes()).hexdigest()
+    else:
+        options_file = None
+
+    header_file = proto_file_without_ext + ".pb.h"
+    source_file = proto_file_without_ext + ".pb.c"
+
+    header_file_abs = os.path.join(generated_src_dir, source_file)
+    source_file_abs = os.path.join(generated_src_dir, header_file)
+
+    need_generate = False
+
+    # Check proto file md5
+    try:
+        last_md5 = pathlib.Path(proto_file_md5_abs).read_text()
+        if last_md5 != proto_file_current_md5:
+            need_generate = True
+    except FileNotFoundError:
+        need_generate = True
+
+    if options_file:
+        # Check options file md5
+        try:
+            last_md5 = pathlib.Path(options_file_md5_abs).read_text()
+            if last_md5 != options_file_current_md5:
+                need_generate = True
+        except FileNotFoundError:
+            need_generate = True
+
+    options_info = f"{options_file}" if options_file else "no options"
+
+    if not need_generate:
+        print(f"[nanopb] Skipping '{proto_file}' ({options_info})")
+    else:
+        print(f"[nanopb] Processing '{proto_file}' ({options_info})")
+        cmd = protoc_generator + " " + nanopb_options + " " + proto_file_basename
+        result = env.Execute(cmd)
+        if result != 0:
+            print(f"[nanopb] ERROR: ({result}) processing cmd: '{cmd}'")
+            exit(1)
+        pathlib.Path(proto_file_md5_abs).write_text(proto_file_current_md5)
+        if options_file:
+            pathlib.Path(options_file_md5_abs).write_text(options_file_current_md5)
+
+#
+# Add generated includes and sources to build environment
+#
+env.Append(CPPPATH=[generated_src_dir])
+env.BuildSources(generated_build_dir, generated_src_dir)

library.json

@@ -1,6 +1,6 @@
 {
   "name": "Nanopb",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "keywords": "protocol buffers, protobuf, google",
   "description": "Nanopb is a plain-C implementation of Google's Protocol Buffers data format. It is targeted at 32 bit microcontrollers, but is also fit for other embedded systems with tight (<10 kB ROM, <1 kB RAM) memory constraints.",
   "repository": {
@@ -17,10 +17,26 @@
       "*.c",
       "*.cpp",
       "*.h",
-      "examples"
+      "examples",
+      "generator"
+    ],
+    "exclude": [
+      "generator/**/__pycache__",
+      "examples/platformio/.gitignore"
     ]
   },
-  "examples": "examples/*/*.c",
+  "build": {
+    "extraScript": "generator/platformio_generator.py",
+    "srcDir": "",
+    "srcFilter": [
+      "+<*.c>"
+    ]
+  },
+  "examples": [
+    "examples/platformio/platformio.ini",
+    "examples/platformio/src/*.c",
+    "examples/*/*.c"
+  ],
   "frameworks": "*",
   "platforms": "*"
 }