archiver/
archive_format.rs

1//! On-disk format shared by the archiver and the recompress tool.
2//!
3//! The `archiver` binary writes every sample file with this schema and these writer
4//! settings, and the `recompress` tool rewrites older files into the same format. The
5//! exporter reads the files back and serves them as CSV over NATS. Keeping the format in
6//! one module means both writers produce identical files.
7
8use parquet::{
9    basic::{Compression, Encoding, ZstdLevel},
10    file::{metadata::KeyValue, properties::WriterProperties},
11    schema::types::ColumnPath,
12};
13
14/// Two-column schema of every archived sample file.
15///
16/// `timestamp_unix_ns` is the sample time in Unix nanoseconds and `value` is the raw
17/// (uncalibrated) sample value. Both columns are required.
18pub const SAMPLE_SCHEMA: &str = "
19    message schema {
20        REQUIRED INT64 timestamp_unix_ns;
21        REQUIRED DOUBLE value;
22    }
23";
24
25/// Parquet writer settings for the two-column sample schema.
26///
27/// Timestamps are strictly regular, so delta encoding reduces them to almost
28/// nothing. Values come from a 16-bit converter and repeat heavily, so they
29/// keep dictionary encoding. zstd compresses the result. Measured on a real
30/// 2 kHz MU1 file this is about 14x smaller than uncompressed PLAIN output.
31///
32/// In detail: zstd at the `parquet` crate's default level for all columns, dictionary
33/// encoding off and `DELTA_BINARY_PACKED` on for `timestamp_unix_ns`, and the writer's
34/// defaults for `value`.
35///
36/// # Arguments
37///
38/// * `key_value_metadata` - File-level key-value metadata to store in the footer, for
39///   example the calibration entry.
40pub fn writer_properties(key_value_metadata: Vec<KeyValue>) -> WriterProperties {
41    let timestamp = ColumnPath::from("timestamp_unix_ns");
42    WriterProperties::builder()
43        .set_compression(Compression::ZSTD(ZstdLevel::default()))
44        .set_column_dictionary_enabled(timestamp.clone(), false)
45        .set_column_encoding(timestamp, Encoding::DELTA_BINARY_PACKED)
46        .set_key_value_metadata(Some(key_value_metadata))
47        .build()
48}
49
50/// Writer settings for a file carrying one calibration JSON document.
51///
52/// Same as [`writer_properties`] with a single metadata entry under the key
53/// `calibration`. Used by the archiver; `dead_code` is allowed because the recompress
54/// tool, which also includes this module, does not call it.
55///
56/// # Arguments
57///
58/// * `calibration_json` - Serialized calibration spec for the file's channel.
59#[allow(dead_code)]
60pub fn writer_properties_for_calibration(calibration_json: String) -> WriterProperties {
61    writer_properties(vec![KeyValue::new(
62        "calibration".to_string(),
63        calibration_json,
64    )])
65}
archiver/archive_format.rs

archiver/
archive_format.rs
