erf_provenance Namespace Reference

Classes
struct	ExecutionProvenance
struct	ArtifactProvenance
struct	ProvenanceRecord
struct	ProvenanceParseResult

Typedefs
using	ProvenanceReadResult = ProvenanceParseResult

Enumerations
enum class	LineageStatus {   Complete , IncompleteAncestor , MissingJobInfo , MissingProvenanceBlock ,   MalformedProvenance , UnsupportedSchema , ArtifactTypeMismatch }
enum class	ArtifactType { Checkpoint , Plotfile2D , Plotfile3D }
enum class	ProvenanceReadStatus {   Valid , MissingJobInfo , MissingProvenanceBlock , MalformedProvenance ,   UnsupportedSchema , ArtifactTypeMismatch }

Functions
std::string	uuid_v4_from_bytes (const std::array< std::uint8_t, 16 > &bytes)
bool	is_valid_uuid_v4 (std::string_view uuid)
std::string	generate_uuid_v4 ()
std::string	format_utc (std::time_t time_value)
std::string	current_utc ()
const char *	lineage_status_token (LineageStatus status) noexcept
const char *	artifact_type_token (ArtifactType type) noexcept
std::string	serialize_provenance_block (const ProvenanceRecord &record)
ProvenanceParseResult	parse_provenance_block (std::string_view text)
ProvenanceReadResult	read_job_info_file (const std::string &path)
ExecutionProvenance	make_cold_start_provenance (const std::string &execution_uuid, const std::string &execution_start_utc)
ExecutionProvenance	make_restart_provenance (const ExecutionProvenance &current_invocation, const ProvenanceRecord &parent_checkpoint, const std::string &checkpoint_path)
ExecutionProvenance	make_incomplete_restart_provenance (const ExecutionProvenance &current_invocation, ProvenanceReadStatus failure_status, const std::string &checkpoint_path)
ExecutionProvenance	initialize_execution_provenance ()

Variables
constexpr int	schema_version = 1
constexpr const char *	provenance_begin = "ERF_PROVENANCE_BEGIN"
constexpr const char *	provenance_end = "ERF_PROVENANCE_END"

Typedef Documentation

ProvenanceReadResult

using erf_provenance::ProvenanceReadResult = typedef ProvenanceParseResult

Enumeration Type Documentation

ArtifactType

enum erf_provenance::ArtifactType

strong

Enumerator
Checkpoint
Plotfile2D
Plotfile3D

{
    Checkpoint,
    Plotfile2D,
    Plotfile3D
};

LineageStatus

enum erf_provenance::LineageStatus

strong

The reason a lineage is incomplete is part of the persisted contract. Missing or invalid metadata never prevents ERF from reading physical state.

Enumerator
Complete
IncompleteAncestor
MissingJobInfo
MissingProvenanceBlock
MalformedProvenance
UnsupportedSchema
ArtifactTypeMismatch

{
    Complete,
    IncompleteAncestor,
    MissingJobInfo,
    MissingProvenanceBlock,
    MalformedProvenance,
    UnsupportedSchema,
    ArtifactTypeMismatch
};

ProvenanceReadStatus

enum erf_provenance::ProvenanceReadStatus

strong

Enumerator
Valid
MissingJobInfo
MissingProvenanceBlock
MalformedProvenance
UnsupportedSchema
ArtifactTypeMismatch

{
    Valid,
    MissingJobInfo,
    MissingProvenanceBlock,
    MalformedProvenance,
    UnsupportedSchema,
    ArtifactTypeMismatch
};

Function Documentation

artifact_type_token()

const char * erf_provenance::artifact_type_token ( ArtifactType  type )

noexcept

Return the stable persisted token for an artifact type.

{
    switch (type) {
    case ArtifactType::Checkpoint: return "checkpoint";
    case ArtifactType::Plotfile2D: return "plotfile_2d";
    case ArtifactType::Plotfile3D: return "plotfile_3d";
    }
    return "checkpoint";
}

Referenced by ERF::ReadCheckpointFile(), serialize_provenance_block(), and ERF::writeJobInfo().

Here is the caller graph for this function:

current_utc()

std::string erf_provenance::current_utc

(

)

Return the current time in the persisted UTC format.

{
    return format_utc(std::time(nullptr));
}

Referenced by initialize_execution_provenance(), and ERF::writeJobInfo().

Here is the call graph for this function:

Here is the caller graph for this function:

format_utc()

std::string erf_provenance::format_utc

(

std::time_t

time_value

)

Format a supplied clock value as an ISO 8601 UTC timestamp.

{
    std::tm utc{};
#if defined(_WIN32)
    gmtime_s(&utc, &time_value);
#else
    gmtime_r(&time_value, &utc);
#endif
    std::ostringstream out;
    out << std::put_time(&utc, "%Y-%m-%dT%H:%M:%SZ");
    return out.str();
}

Referenced by current_utc().

Here is the caller graph for this function:

generate_uuid_v4()

std::string erf_provenance::generate_uuid_v4

(

)

Generate a UUID without using AMReX's scientific random-number stream.

{
    std::array<std::uint8_t, 16> bytes{};
    bool generated = false;
    try {
        std::random_device random;
        for (auto& byte : bytes) byte = static_cast<std::uint8_t>(random());
        generated = true;
    } catch (...) {
        generated = false;
    }
    const auto now = std::chrono::high_resolution_clock::now().time_since_epoch().count();
    const auto steady = std::chrono::steady_clock::now().time_since_epoch().count();
    const auto address = reinterpret_cast<std::uintptr_t>(&bytes);
    if (!generated) {
        for (std::size_t i = 0; i < bytes.size(); ++i) {
            bytes[i] = static_cast<std::uint8_t>((now >> ((i % 8) * 8)) ^
                                                 (steady >> (((i + 3) % 8) * 8)) ^
                                                 (address >> ((i % sizeof(std::uintptr_t)) * 8)));
        }
    } else {
        for (std::size_t i = 0; i < bytes.size(); ++i) {
            bytes[i] ^= static_cast<std::uint8_t>((now >> ((i % 8) * 8)) ^
                                                   (steady >> (((i + 3) % 8) * 8)) ^
                                                   (address >> ((i % sizeof(std::uintptr_t)) * 8)));
        }
    }
    bytes[6] = static_cast<std::uint8_t>((bytes[6] & 0x0f) | 0x40);
    bytes[8] = static_cast<std::uint8_t>((bytes[8] & 0x3f) | 0x80);
    return uuid_v4_from_bytes(bytes);
}

Referenced by initialize_execution_provenance(), and ERF::writeJobInfo().

Here is the call graph for this function:

Here is the caller graph for this function:

initialize_execution_provenance()

ExecutionProvenance erf_provenance::initialize_execution_provenance

(

)

Initialize one process-wide invocation record and share it across multiblock ERF objects.

{
    static bool initialized = false;
    static ExecutionProvenance invocation;
    if (!initialized) {
        std::string execution_uuid;
        std::string execution_start_utc;
        if (amrex::ParallelDescriptor::IOProcessor()) {
            execution_uuid = generate_uuid_v4();
            execution_start_utc = current_utc();
        }
        broadcast_string(execution_uuid);
        broadcast_string(execution_start_utc);
        invocation = make_cold_start_provenance(execution_uuid, execution_start_utc);
        initialized = true;
    }
    return invocation;
}

Referenced by ERF::ERF_shared().

Here is the call graph for this function:

Here is the caller graph for this function:

is_valid_uuid_v4()

bool erf_provenance::is_valid_uuid_v4

(

std::string_view

uuid

)

Validate a lowercase or uppercase canonical RFC UUIDv4 string.

{
    if (!is_canonical_uuid_layout(uuid) || uuid[14] != '4') return false;
    const char variant = uuid[19];
    return variant == '8' || variant == '9' || variant == 'a' || variant == 'b' ||
           variant == 'A' || variant == 'B';
}

lineage_status_token()

const char * erf_provenance::lineage_status_token ( LineageStatus  status )

noexcept

Return the stable persisted token for a lineage status.

{
    switch (status) {
    case LineageStatus::Complete: return "complete";
    case LineageStatus::IncompleteAncestor: return "incomplete_ancestor";
    case LineageStatus::MissingJobInfo: return "missing_job_info";
    case LineageStatus::MissingProvenanceBlock: return "missing_provenance_block";
    case LineageStatus::MalformedProvenance: return "malformed_provenance";
    case LineageStatus::UnsupportedSchema: return "unsupported_schema";
    case LineageStatus::ArtifactTypeMismatch: return "artifact_type_mismatch";
    }
    return "malformed_provenance";
}

Referenced by serialize_provenance_block().

Here is the caller graph for this function:

make_cold_start_provenance()

ExecutionProvenance erf_provenance::make_cold_start_provenance	(	const std::string &	execution_uuid,
		const std::string &	execution_start_utc
	)

Create the root record for one cold-start invocation.

{
    ExecutionProvenance result;
    result.simulation_uuid = execution_uuid;
    result.execution_uuid = execution_uuid;
    result.execution_lineage = {execution_uuid};
    result.execution_start_utc = execution_start_utc;
    return result;
}

Referenced by initialize_execution_provenance().

Here is the caller graph for this function:

make_incomplete_restart_provenance()

ExecutionProvenance erf_provenance::make_incomplete_restart_provenance	(	const ExecutionProvenance &	current_invocation,
		ProvenanceReadStatus	failure,
		const std::string &	checkpoint_path
	)

Start a new known lineage segment after a nonfatal provenance read failure.

{
    ExecutionProvenance result = current_invocation;
    result.simulation_uuid = current_invocation.execution_uuid;
    result.parent_execution_uuid.clear();
    result.execution_lineage = {current_invocation.execution_uuid};
    result.restart_generation = -1;
    result.lineage_complete = false;
    result.lineage_status = failure_lineage_status(failure_status);
    result.source_checkpoint_uuid.clear();
    result.source_checkpoint_path = checkpoint_path;
    return result;
}

Referenced by ERF::ReadCheckpointFile().

Here is the caller graph for this function:

make_restart_provenance()

ExecutionProvenance erf_provenance::make_restart_provenance	(	const ExecutionProvenance &	current_invocation,
		const ProvenanceRecord &	parent_checkpoint,
		const std::string &	checkpoint_path
	)

Create a child record from a valid checkpoint record, including its artifact UUID.

{
    ExecutionProvenance result = current_invocation;
    const auto& parent = parent_checkpoint.execution;
    result.simulation_uuid = parent.simulation_uuid;
    result.parent_execution_uuid = parent.execution_uuid;
    result.execution_lineage = parent.execution_lineage;
    result.execution_lineage.push_back(current_invocation.execution_uuid);
    result.source_checkpoint_uuid = parent_checkpoint.artifact.artifact_uuid;
    result.source_checkpoint_path = checkpoint_path;
    if (parent.lineage_complete) {
        result.restart_generation = parent.restart_generation + 1;
        result.lineage_complete = true;
        result.lineage_status = LineageStatus::Complete;
    } else {
        result.restart_generation = -1;
        result.lineage_complete = false;
        result.lineage_status = LineageStatus::IncompleteAncestor;
    }
    return result;
}

Referenced by ERF::ReadCheckpointFile().

Here is the caller graph for this function:

parse_provenance_block()

ProvenanceParseResult erf_provenance::parse_provenance_block

(

std::string_view

job_info_text

)

Parse only delimited provenance markers from a human-readable job_info.

{
    enum class BlockState { BeforeBlock, InsideBlock, AfterBlock };
    BlockState state = BlockState::BeforeBlock;
    std::vector<std::string> block_lines;
    std::istringstream input{std::string(text)};
    std::string line;
    while (std::getline(input, line)) {
        line = strip_cr(std::move(line));
        const bool is_begin = line == provenance_begin;
        const bool is_end = line == provenance_end;
        if (state == BlockState::BeforeBlock) {
            if (is_end) {
                return failure(ProvenanceReadStatus::MalformedProvenance,
                               "provenance end marker precedes begin marker");
            }
            if (is_begin) state = BlockState::InsideBlock;
        } else if (state == BlockState::InsideBlock) {
            if (is_begin) {
                return failure(ProvenanceReadStatus::MalformedProvenance,
                               "provenance block has a nested begin marker");
            }
            if (is_end) {
                state = BlockState::AfterBlock;
            } else {
                block_lines.push_back(line);
            }
        } else if (is_begin || is_end) {
            return failure(ProvenanceReadStatus::MalformedProvenance,
                           "job_info has multiple provenance blocks");
        }
    }
    if (state == BlockState::BeforeBlock) {
        return failure(ProvenanceReadStatus::MissingProvenanceBlock,
                       "job_info has no exact ERF provenance block");
    }
    if (state == BlockState::InsideBlock) {
        return failure(ProvenanceReadStatus::MalformedProvenance,
                       "provenance block has no matching end marker");
    }
 
    std::unordered_map<std::string, std::string> fields;
    static const std::unordered_set<std::string> required = {
        "schema_version", "simulation_uuid", "execution_uuid", "parent_execution_uuid",
        "execution_lineage", "restart_generation", "lineage_complete", "lineage_status",
        "source_checkpoint_uuid", "source_checkpoint_path", "execution_start_utc",
        "artifact_uuid", "artifact_type", "artifact_step", "artifact_time_seconds",
        "artifact_created_utc"
    };
    for (const auto& block_line : block_lines) {
        if (block_line.empty()) continue;
        const std::size_t equals = block_line.find('=');
        if (equals == std::string::npos || equals == 0) {
            return failure(ProvenanceReadStatus::MalformedProvenance,
                           "provenance line is not a key=value pair");
        }
        const std::string key = block_line.substr(0, equals);
        if (required.count(key) != 0 && fields.count(key) != 0) {
            return failure(ProvenanceReadStatus::MalformedProvenance,
                           "duplicate required provenance key: " + key);
        }
        if (required.count(key) != 0) fields.emplace(key, block_line.substr(equals + 1));
    }
    ProvenanceParseResult result;
    int schema = 0;
    if (fields.count("schema_version") == 0 ||
        !parse_int(fields.at("schema_version"), schema)) {
        return failure(ProvenanceReadStatus::MalformedProvenance, "invalid provenance schema_version");
    }
    if (schema != erf_provenance::schema_version) {
        return failure(ProvenanceReadStatus::UnsupportedSchema, "unsupported provenance schema_version");
    }
    for (const auto& key : required) {
        if (fields.count(key) == 0) {
            return failure(ProvenanceReadStatus::MalformedProvenance,
                           "missing required provenance key: " + key);
        }
    }
    result.record.execution.schema_version = schema;
    result.record.execution.simulation_uuid = fields.at("simulation_uuid");
    result.record.execution.execution_uuid = fields.at("execution_uuid");
    result.record.execution.parent_execution_uuid = fields.at("parent_execution_uuid");
    result.record.execution.execution_lineage = split_lineage(fields.at("execution_lineage"));
    if (result.record.execution.execution_lineage.empty()) {
        return failure(ProvenanceReadStatus::MalformedProvenance, "invalid execution_lineage");
    }
    if (!parse_int(fields.at("restart_generation"), result.record.execution.restart_generation) ||
        !parse_bool(fields.at("lineage_complete"), result.record.execution.lineage_complete) ||
        !parse_lineage_status(fields.at("lineage_status"), result.record.execution.lineage_status) ||
        !parse_artifact_type(fields.at("artifact_type"), result.record.artifact.artifact_type) ||
        !parse_int(fields.at("artifact_step"), result.record.artifact.artifact_step) ||
        !parse_double(fields.at("artifact_time_seconds"), result.record.artifact.artifact_time_seconds)) {
        return failure(ProvenanceReadStatus::MalformedProvenance, "invalid typed provenance field");
    }
    result.record.execution.source_checkpoint_uuid = fields.at("source_checkpoint_uuid");
    result.record.execution.source_checkpoint_path = fields.at("source_checkpoint_path");
    result.record.execution.execution_start_utc = fields.at("execution_start_utc");
    result.record.artifact.artifact_uuid = fields.at("artifact_uuid");
    result.record.artifact.artifact_created_utc = fields.at("artifact_created_utc");
 
    std::string diagnostic;
    if (!validate_record(result.record, diagnostic)) {
        return failure(ProvenanceReadStatus::MalformedProvenance, std::move(diagnostic));
    }
    result.status = ProvenanceReadStatus::Valid;
    return result;
}

Referenced by read_job_info_file().

Here is the caller graph for this function:

read_job_info_file()

ProvenanceReadResult erf_provenance::read_job_info_file

(

const std::string &

job_info_path

)

Read and broadcast one job_info before parsing the same bytes on every rank.

{
    int read_status = static_cast<int>(ProvenanceReadStatus::Valid);
    std::string contents;
    if (amrex::ParallelDescriptor::IOProcessor()) {
        std::ifstream input(path, std::ios::in | std::ios::binary);
        if (!input.good()) {
            read_status = static_cast<int>(ProvenanceReadStatus::MissingJobInfo);
        } else {
            contents.assign(std::istreambuf_iterator<char>(input), std::istreambuf_iterator<char>());
            if (!input.good() && !input.eof()) {
                read_status = static_cast<int>(ProvenanceReadStatus::MalformedProvenance);
            }
        }
    }
    const int io_rank = amrex::ParallelDescriptor::IOProcessorNumber();
    amrex::ParallelDescriptor::Bcast(&read_status, 1, io_rank);
    broadcast_string(contents);
    if (read_status != static_cast<int>(ProvenanceReadStatus::Valid)) {
        return failure(static_cast<ProvenanceReadStatus>(read_status),
                       read_status == static_cast<int>(ProvenanceReadStatus::MissingJobInfo)
                           ? "job_info does not exist" : "job_info could not be read");
    }
    return parse_provenance_block(contents);
}

Referenced by ERF::ReadCheckpointFile().

Here is the call graph for this function:

Here is the caller graph for this function:

serialize_provenance_block()

std::string erf_provenance::serialize_provenance_block

(

const ProvenanceRecord &

record

)

Serialize exactly one schema-1 machine-readable block.

{
    std::ostringstream out;
    out << provenance_begin << '\n';
    out << "schema_version=" << record.execution.schema_version << '\n';
    out << "simulation_uuid=" << record.execution.simulation_uuid << '\n';
    out << "execution_uuid=" << record.execution.execution_uuid << '\n';
    out << "parent_execution_uuid=" << record.execution.parent_execution_uuid << '\n';
    out << "execution_lineage=";
    for (std::size_t i = 0; i < record.execution.execution_lineage.size(); ++i) {
        if (i != 0) out << ',';
        out << record.execution.execution_lineage[i];
    }
    out << '\n';
    out << "restart_generation=" << record.execution.restart_generation << '\n';
    out << "lineage_complete=" << (record.execution.lineage_complete ? "true" : "false") << '\n';
    out << "lineage_status=" << lineage_status_token(record.execution.lineage_status) << '\n';
    out << "source_checkpoint_uuid=" << record.execution.source_checkpoint_uuid << '\n';
    out << "source_checkpoint_path=" << record.execution.source_checkpoint_path << '\n';
    out << "execution_start_utc=" << record.execution.execution_start_utc << '\n';
    out << "artifact_uuid=" << record.artifact.artifact_uuid << '\n';
    out << "artifact_type=" << artifact_type_token(record.artifact.artifact_type) << '\n';
    out << "artifact_step=" << record.artifact.artifact_step << '\n';
    out << "artifact_time_seconds=" << std::setprecision(17)
        << record.artifact.artifact_time_seconds << '\n';
    out << "artifact_created_utc=" << record.artifact.artifact_created_utc << '\n';
    out << provenance_end << '\n';
    return out.str();
}

Referenced by ERF::writeJobInfo().

Here is the call graph for this function:

Here is the caller graph for this function:

uuid_v4_from_bytes()

std::string erf_provenance::uuid_v4_from_bytes

(

const std::array< std::uint8_t, 16 > &

bytes

)

Format supplied bytes as a lowercase canonical RFC UUIDv4 string.

{
    static constexpr char digits[] = "0123456789abcdef";
    std::string result;
    result.reserve(36);
    for (std::size_t i = 0; i < bytes.size(); ++i) {
        if (i == 4 || i == 6 || i == 8 || i == 10) result.push_back('-');
        result.push_back(digits[(bytes[i] >> 4) & 0x0f]);
        result.push_back(digits[bytes[i] & 0x0f]);
    }
    return result;
}

Referenced by generate_uuid_v4().

Here is the caller graph for this function:

Variable Documentation

provenance_begin

constexpr const char* erf_provenance::provenance_begin = "ERF_PROVENANCE_BEGIN"

constexpr

Referenced by parse_provenance_block(), and serialize_provenance_block().

provenance_end

constexpr const char* erf_provenance::provenance_end = "ERF_PROVENANCE_END"

constexpr

Referenced by parse_provenance_block(), and serialize_provenance_block().

schema_version

constexpr int erf_provenance::schema_version = 1

constexpr

Referenced by parse_provenance_block().