Explore ways to transfer structured data between the apio process and the scons process

[zapta]

The scons doesn't access data such as apio project, profile and resources data and recieves all of its data via a flat Dict[str, str]. This issue is for exploring a more structured way of transferring data from the apio process to the scons process.

Things to look for (not all are absolutely required):

• Native support of types.
• Declared schema
• Automatic serialization/deserialization
• Human readable wire format
• Data size is non restricted (e.g. by command line size)
• Support for optional with defaults and required values.
• Constructs such as enum, structs, and variants.
• Time efficiency.

One candidate that comes to mind is Protocol Buffers 2 with text serialization. The apio process can write it to a file in _build and the scons process can read it from there. Since the serialization used internally within a single apio invocation, there are no issues of backward compatibility.

[zapta]

Here is a quick feasibility test with protocol buffer 2 in text mode. Scons params are serialized to a file and are read and parsed back. Zip file with the experiment files at the bottom of this message.

Protocol buffer definition

syntax = "proto2";

package apio.proto;

// The supported FPGA architectures, each with its own handler.
enum ApioArch{
  UNSPECIFIED = 0;
  ICE40 = 1;
  ECP5 = 2;
  GOWIN = 3;
}


// Ice40 specific fpga attributes.
message Ice40FpgaInfo {
  required string fpga_id = 1;
  required string type = 4;
  required string pack = 5;
}

// Ecp5 specific fpga attributes.
message Ecp5FpgaInfo {
  required string type = 4;
  required string pack = 5;
  required uint32 speed = 6;
}

// Gowin specific fpga attributes.
message GowinFpgaInfo {
  required string family = 4;
}

// Ice40 fpga info.
message FpgaInfo {
  // Common fpga attributes.
  required string fpga_id = 1;
  required string part_num = 2;
  required string size = 3;
  // Architecture specific fpga attributes.
  oneof arch {
    Ice40FpgaInfo ice40 = 10;
    Ecp5FpgaInfo ecp5 = 11;
    GowinFpgaInfo gowin = 12;
  }
}

// The top level messages that is passed from the apio process to
// the scons process.
message SconsParams {
   required ApioArch arch = 1;
   required string board_id = 2;
   required FpgaInfo fpga_info = 3;

   // ... more fields here.  E.g. per command params.

}

Python test code

from google.protobuf import text_format
from apio_pb2 import (
    SconsParams,
    ApioArch,
    FpgaInfo,
    Ecp5FpgaInfo,
)

# Initialize a top message.
scons_params1 = SconsParams(
    arch=ApioArch.ECP5,
    board_id="colorlight-5a-75e-v6",
    fpga_info=FpgaInfo(
        fpga_id="lfe5u-25f-6bg256c",
        part_num="LFE5U-25F-6BG256C",
        size="25k",
        ecp5=Ecp5FpgaInfo(type="25k", pack="CABGA256", speed=6),
    ),
)

# Check that all the required fields where populated.
assert scons_params1.IsInitialized()

# Serialize the top message in TEXT mode.
print("\nscons_params1:")
print(scons_params1)


# Write the serialization to a file.
with open("scons.pb_text", "w") as text_file:
    text_file.write(
        "# Auto generated scons params\n\n" + text_format.MessageToString(scons_params1)
    )

# Read the text back from the file
with open("scons.pb_text", "r") as f:
    text = f.read()

# Deserialize the text to a SconsParams object
scons_params2 = text_format.Parse(text, SconsParams())

# Check that all the required fields are populated and that the object
# equals the original one.
assert scons_params1.IsInitialized()
assert scons_params2 == scons_params1


# print("\nscons_params2:")
print(scons_params2)

# Access fields
print(f"{scons_params2.board_id=}")
print(f"{scons_params2.arch=}")
print(f"{scons_params2.fpga_info.ecp5.speed=}")


# print(text_format.MessageToString(a))

# Parse a text proto string.
# message = text_format.Parse(text_proto, my_proto_pb2.MyMessage())

Generated serialized data

# Auto generated scons params

arch: ECP5
board_id: "colorlight-5a-75e-v6"
fpga_info {
  fpga_id: "lfe5u-25f-6bg256c"
  part_num: "LFE5U-25F-6BG256C"
  size: "25k"
  ecp5 {
    type: "25k"
    pack: "CABGA256"
    speed: 6
  }
}

work-proto.zip

[zapta]

Here is a more complete draft of the Protocol Buffers definitions for for the scons params. Not tested or integrated yet.

// Protocol buffers definitions used by apio.  Currently we use them just
// for passing parameters from the apio process to the scons process.


// Online proto formatter at https://formatter.org/protobuf-formatter

// NOTE: Since we use the the serialized proto data within a single invocation
// of apio, protocol buffers text mode and binary mode backeard compatibility
// considerations do not apply.

// Using proto2 for features such as 'has' and 'required'. 
syntax = "proto2";


package apio.proto;

// The supported FPGA architectures, each with its own handler.
enum ApioArch {
  UNSPECIFIED = 0;
  ICE40 = 1;
  ECP5 = 2;
  GOWIN = 3;
}

// Ice40 specific fpga attributes.
message Ice40FpgaInfo {
  required string fpga_id = 1;
  required string type = 4;
  required string pack = 5;
}

// Ecp5 specific fpga attributes.
message Ecp5FpgaInfo {
  required string type = 4;
  required string pack = 5;
  required uint32 speed = 6;
}

// Gowin specific fpga attributes.
message GowinFpgaInfo {
  required string family = 4;
}

// General fpga info.
message FpgaInfo {
  // Common fpga attributes.
  required string fpga_id = 1;
  required string part_num = 2;
  required string size = 3;
  // Architecture specific fpga attributes.
  oneof arch {
    Ice40FpgaInfo ice40 = 10;
    Ecp5FpgaInfo ecp5 = 11;
    GowinFpgaInfo gowin = 12;
  }
}

// Verbosity levels.
message Verbosity {
  // If true, enable general verbosity.
  optional bool all = 1 [default = false];
  // If true, enable synthesis verbosiry.
  optional bool synth = 2 [default = false];
  // If true, enable place-and-route verbosity.
  optional bool pnr = 3 [default = false];
}

// Information about the environment.
message Envrionment {
  // The underlying platform id as it appears in platforms.jsonc.
  required string platform_id = 1;

  // True if apio debug is enabled.
  optional bool debug = 2 [default = false];

  // Paths to tools.
  required string yosys_path = 3;
  required string trellis_path = 4;
}

// Information about the project.
message Project {
  required string board_id = 1;
  required string top_module = 2;
}

// Lint command specific info.
message CmdLintInfo {
  optional bool verilator_all = 1 [default = false];
  optional bool verilator_no_style = 2 [default = false];
  repeated string verilator_no_warn = 3;
  repeated string verilator_warn = 4;
}

// Graph command specific info.
message CmdGraphInfo {
  required string graph_spec = 1;
}

// Sim command specific info.
message CmdSimInfo {
  optional string test_bench = 1;
}

// Test command specific info.
message CmdTestInfo {
  optional string test_bench = 1;
}

// The top level messages that is passed from the apio process to
// the scons process.
message SconsParams {
  // The apio architecture of this project. Used to select the scons
  // architecture plugin.
  required ApioArch arch = 1;

  // Information about the FPGA used in the project. From fpgas.jsonc.
  required FpgaInfo fpga_info = 3;

  // Verbosity flags.
  optional Verbosity verbosity = 4;

  // General information about the environment.
  required Envrionment envrionment = 5;

  // General information about the project
  required Project project = 6;

  // Some commands require additional command-specific information.
  oneof cmd {
    CmdLintInfo cmd_lint = 10;
    CmdGraphInfo cmd_graph = 11;
    CmdSimInfo cmd_sim = 12;
    CmdTestInfo cmd_test = 13;
  }
}