8000 feat: pass data directly in cli by alexander-camuto · Pull Request #939 · zkonduit/ezkl · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

feat: pass data directly in cli #939

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 13, 2025
Merged

feat: pass data directly in cli #939

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c353043
feat: pass data directly in cli
alexander-camuto Feb 12, 2025
d748bfc
Update ezkl.rs
alexander-camuto Feb 12, 2025
57a123f
Update python.rs
alexander-camuto Feb 12, 2025
f78e855
Update python.rs
alexander-camuto Feb 12, 2025
e4af50e
Update python.rs
alexander-camuto Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions examples/onnx/1l_div/settings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"run_args":{"input_scale":7,"param_scale":7,"scale_rebase_multiplier":1,"lookup_range":[-32768,32768],"logrows":17,"num_inner_cols":2,"variables":[["batch_size",1]],"input_visibility":"Private","output_visibility":"Public","param_visibility":"Private","rebase_frac_zero_constants":false,"check_mode":"UNSAFE","commitment":"KZG","decomp_base":16384,"decomp_legs":2,"bounded_log_lookup":false,"ignore_range_check_inputs_outputs":false},"num_rows":54,"total_assignments":109,"total_const_size":4,"total_dynamic_col_size":0,"max_dynamic_input_len":0,"num_dynamic_lookups":0,"num_shuffles":0,"total_shuffle_col_size":0,"model_instance_shapes":[[1,1]],"model_output_scales":[7],"model_input_scales":[7],"module_sizes":{"polycommit":[],"poseidon":[0,[0]]},"required_lookups":[],"required_range_checks":[[-1,1],[0,16383]],"check_mode":"UNSAFE","version":"0.0.0","num_blinding_factors":null,"timestamp":1739396322131,"input_types":["F32"],"output_types":["F32"]}
4 changes: 3 additions & 1 deletion src/bin/ezkl.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ use std::env;
#[tokio::main(flavor = "current_thread")]
#[cfg(all(feature = "ezkl", not(target_arch = "wasm32")))]
pub async fn main() {
use log::debug;

let args = Cli::parse();

if let Some(generator) = args.generator {
Expand All @@ -42,7 +44,7 @@ pub async fn main() {
} else {
info!("Running with CPU");
}
info!(
debug!(
"command: \n {}",
&command.as_json().to_colored_json_auto().unwrap()
);
Expand Down
16 changes: 8 additions & 8 deletions src/bindings/python.rs
10000
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1009,7 +1009,7 @@ fn gen_random_data(
/// bool
///
#[pyfunction(signature = (
data = PathBuf::from(DEFAULT_CALIBRATION_FILE),
data = String::from(DEFAULT_CALIBRATION_FILE),
model = PathBuf::from(DEFAULT_MODEL),
settings = PathBuf::from(DEFAULT_SETTINGS),
target = CalibrationTarget::default(), // default is "resources
Expand All @@ -1021,7 +1021,7 @@ fn gen_random_data(
#[gen_stub_pyfunction]
fn calibrate_settings(
py: Python,
data: PathBuf,
data: String,
model: PathBuf,
settings: PathBuf,
target: CalibrationTarget,
Expand Down Expand Up @@ -1076,7 +1076,7 @@ fn calibrate_settings(
/// Python object containing the witness values
///
#[pyfunction(signature = (
data=PathBuf::from(DEFAULT_DATA),
data=String::from(DEFAULT_DATA),
model=PathBuf::from(DEFAULT_COMPILED_CIRCUIT),
output=PathBuf::from(DEFAULT_WITNESS),
vk_path=None,
Expand All @@ -1085,7 +1085,7 @@ fn calibrate_settings(
#[gen_stub_pyfunction]
fn gen_witness(
py: Python,
data: PathBuf,
data: String,
model: PathBuf,
output: Option<PathBuf>,
vk_path: Option<PathBuf>,
Expand Down Expand Up @@ -1754,7 +1754,7 @@ fn create_evm_vka(
/// bool
///
#[pyfunction(signature = (
input_data=PathBuf::from(DEFAULT_DATA),
input_data=String::from(DEFAULT_DATA),
settings_path=PathBuf::from(DEFAULT_SETTINGS),
sol_code_path=PathBuf::from(DEFAULT_SOL_CODE_DA),
abi_path=PathBuf::from(DEFAULT_VERIFIER_DA_ABI),
Expand All @@ -1763,7 +1763,7 @@ fn create_evm_vka(
#[gen_stub_pyfunction]
fn create_evm_data_attestation(
py: Python,
input_data: PathBuf,
input_data: String,
settings_path: PathBuf,
sol_code_path: PathBuf,
abi_path: PathBuf,
Expand Down Expand Up @@ -1824,7 +1824,7 @@ fn create_evm_data_attestation(
#[gen_stub_pyfunction]
fn setup_test_evm_witness(
py: Python,
data_path: PathBuf,
data_path: String,
compiled_circuit_path: PathBuf,
test_data: PathBuf,
input_source: PyTestDataSource,
Expand Down Expand Up @@ -1902,7 +1902,7 @@ fn deploy_evm(
fn deploy_da_evm(
py: Python,
addr_path: PathBuf,
input_data: PathBuf,
input_data: String,
settings_path: PathBuf,
sol_code_path: PathBuf,
rpc_url: Option<String>,
Expand Down
19 changes: 12 additions & 7 deletions src/commands.rs
View file
Open in desktop
6D4E
Original file line number Diff line number Diff line change
Expand Up @@ -396,8 +396,9 @@ pub enum Commands {
/// Generates the witness from an input file.
GenWitness {
/// The path to the .json data file
/// You can also pass the input data as a string, eg. --data '{"input_data": [1.0,2.0,3.0]}' directly and skip the file
#[arg(short = 'D', long, default_value = DEFAULT_DATA, value_hint = clap::ValueHint::FilePath)]
data: Option<PathBuf>,
data: Option<String>,
/// The path to the compiled model file (generated using the compile-circuit command)
#[arg(short = 'M', long, default_value = DEFAULT_COMPILED_CIRCUIT, value_hint = clap::ValueHint::FilePath)]
compiled_circuit: Option<PathBuf>,
Expand Down Expand Up @@ -429,7 +430,7 @@ pub enum Commands {
/// The path to the .onnx model file
#[arg(short = 'M', long, default_value = DEFAULT_MODEL, value_hint = clap::ValueHint::FilePath)]
model: Option<PathBuf>,
/// The path to the .json data file
/// The path to the .json data file to output
#[arg(short = 'D', long, default_value = DEFAULT_DATA, value_hint = clap::ValueHint::FilePath)]
data: Option<PathBuf>,
/// Hand-written parser for graph variables, eg. batch_size=1
Expand All @@ -442,8 +443,9 @@ pub enum Commands {
/// Calibrates the proving scale, lookup bits and logrows from a circuit settings file.
CalibrateSettings {
/// The path to the .json calibration data file.
/// You can also pass the input data as a string, eg. --data '{"input_data": [1.0,2.0,3.0]}' directly and skip the file
#[arg(short = 'D', long, default_value = DEFAULT_CALIBRATION_FILE, value_hint = clap::ValueHint::FilePath)]
data: Option<PathBuf>,
data: Option<String>,
/// The path to the .onnx model file
#[arg(short = 'M', long, default_value = DEFAULT_MODEL, value_hint = clap::ValueHint::FilePath)]
model: Option<PathBuf>,
Expand Down Expand Up @@ -626,8 +628,9 @@ pub enum Commands {
#[command(arg_required_else_help = true)]
SetupTestEvmData {
/// The path to the .json data file, which should include both the network input (possibly private) and the network output (public input to the proof)
/// You can also pass the input data as a string, eg. --data '{"input_data": [1.0,2.0,3.0]}' directly and skip the file
#[arg(short = 'D', long, value_hint = clap::ValueHint::FilePath)]
data: Option<PathBuf>,
data: Option<String>,
/// The path to the compiled model file (generated using the compile-circuit command)
#[arg(short = 'M', long, value_hint = clap::ValueHint::FilePath)]
compiled_circuit: Option<PathBuf>,
Expand All @@ -653,8 +656,9 @@ pub enum Commands {
#[arg(long, value_hint = clap::ValueHint::Other)]
addr: H160Flag,
/// The path to the .json data file.
/// You can also pass the input data as a string, eg. --data '{"input_data": [1.0,2.0,3.0]}' directly and skip the file
#[arg(short = 'D', long, value_hint = clap::ValueHint::FilePath)]
data: Option<PathBuf>,
data: Option<String>,
/// RPC URL for an Ethereum node, if None will use Anvil but WON'T persist state
#[arg(short = 'U', long, value_hint = clap::ValueHint::Url)]
rpc_url: Option<String>,
Expand Down Expand Up @@ -771,7 +775,7 @@ pub enum Commands {
/// view functions that return the data that the network
/// ingests as inputs.
#[arg(short = 'D', long, default_value = DEFAULT_DATA, value_hint = clap::ValueHint::FilePath)]
data: Option<PathBuf>,
data: Option<String>,
/// The path to the witness file. This is needed for proof swapping for kzg commitments.
#[arg(short = 'W', long, default_value = DEFAULT_WITNESS, value_hint = clap::ValueHint::FilePath)]
witness: Option<PathBuf>,
Expand Down Expand Up @@ -866,8 +870,9 @@ pub enum Commands {
#[command(name = "deploy-evm-da")]
DeployEvmDataAttestation {
/// The path to the .json data file, which should include both the network input (possibly private) and the network output (public input to the proof)
/// You can also pass the input data as a string, eg. --data '{"input_data": [1.0,2.0,3.0]}' directly and skip the file
#[arg(short = 'D', long, default_value = DEFAULT_DATA, value_hint = clap::ValueHint::FilePath)]
data: Option<PathBuf>,
data: Option<String>,
/// The path to load circuit settings .json file from (generated using the gen-settings command)
#[arg(long, default_value = DEFAULT_SETTINGS, value_hint = clap::ValueHint::FilePath)]
settings_path: Option<PathBuf>,
Expand Down
8 changes: 4 additions & 4 deletions src/eth.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,15 +383,15 @@ pub async fn deploy_contract_via_solidity(
///
pub async fn deploy_da_verifier_via_solidity(
settings_path: PathBuf,
input: PathBuf,
input: String,
sol_code_path: PathBuf,
rpc_url: Option<&str>,
runs: usize,
private_key: Option<&str>,
) -> Result<H160, EthError> {
let (client, client_address) = setup_eth_backend(rpc_url, private_key).await?;

let input = GraphData::from_path(input).map_err(|_| EthError::GraphData)?;
let input = GraphData::from_str(&input).map_err(|_| EthError::GraphData)?;

let settings = GraphSettings::load(&settings_path).map_err(|_| EthError::GraphSettings)?;

Expand Down Expand Up @@ -688,10 +688,10 @@ fn parse_call_to_account(call_to_account: CallToAccount) -> Result<ParsedCallToA

pub async fn update_account_calls(
addr: H160,
input: PathBuf,
input: String,
rpc_url: Option<&str>,
) -> Result<(), EthError> {
let input = GraphData::from_path(input).map_err(|_| EthError::GraphData)?;
let input = GraphData::from_str(&input).map_err(|_| EthError::GraphData)?;

// The data that will be stored in the test contracts that will eventually be read from.
let mut calls_to_accounts = vec![];
Expand Down
20 changes: 10 additions & 10 deletions src/execute.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -725,15 +725,15 @@ pub(crate) fn table(model: PathBuf, run_args: RunArgs) -> Result<String, EZKLErr

pub(crate) async fn gen_witness(
compiled_circuit_path: PathBuf,
data: PathBuf,
data: String,
output: Option<PathBuf>,
vk_path: Option<PathBuf>,
srs_path: Option<PathBuf>,
) -> Result<GraphWitness, EZKLError> {
// these aren't real values so the sanity checks are mostly meaningless

let mut circuit = GraphCircuit::load(compiled_circuit_path)?;
let data: GraphData = GraphData::from_path(data)?;
let data = GraphData::from_str(&data)?;
let settings = circuit.settings().clone();

let vk = if let Some(vk) = vk_path {
Expand Down Expand Up @@ -1044,7 +1044,7 @@ impl AccuracyResults {
#[allow(clippy::too_many_arguments)]
pub(crate) async fn calibrate(
model_path: PathBuf,
data: PathBuf,
data: String,
settings_path: PathBuf,
target: CalibrationTarget,
lookup_safety_margin: f64,
Expand All @@ -1058,7 +1058,7 @@ pub(crate) async fn calibrate(

use crate::fieldutils::IntegerRep;

let data = GraphData::from_path(data)?;
let data = GraphData::from_str(&data)?;
// load the pre-generated settings
let settings = GraphSettings::load(&settings_path)?;
// now retrieve the run args
Expand Down Expand Up @@ -1522,7 +1522,7 @@ pub(crate) async fn create_evm_data_attestation(
settings_path: PathBuf,
sol_code_path: PathBuf,
abi_path: PathBuf,
input: PathBuf,
input: String,
witness: Option<PathBuf>,
) -> Result<String, EZKLError> {
#[allow(unused_imports)]
Expand All @@ -1536,7 +1536,7 @@ pub(crate) async fn create_evm_data_attestation(

// if input is not provided, we just instantiate dummy input data
let data =
GraphData::from_path(input).unwrap_or_else(|_| GraphData::new(DataSource::File(vec![])));
GraphData::from_str(&input).unwrap_or_else(|_| GraphData::new(DataSource::File(vec![])));

// The number of input and output instances we attest to for the single call data attestation
let mut input_len = None;
Expand Down Expand Up @@ -1625,7 +1625,7 @@ pub(crate) async fn create_evm_data_attestation(
}

pub(crate) async fn deploy_da_evm(
data: PathBuf,
data: String,
settings_path: PathBuf,
sol_code_path: PathBuf,
rpc_url: Option<String>,
Expand Down Expand Up @@ -1868,7 +1868,7 @@ pub(crate) fn setup(
}

pub(crate) async fn setup_test_evm_witness(
data_path: PathBuf,
data_path: String,
compiled_circuit_path: PathBuf,
test_data: PathBuf,
rpc_url: Option<String>,
Expand All @@ -1877,7 +1877,7 @@ pub(crate) async fn setup_test_evm_witness(
) -> Result<String, EZKLError> {
use crate::graph::TestOnChainData;

let mut data = GraphData::from_path(data_path)?;
let mut data = GraphData::from_str(&data_path)?;
let mut circuit = GraphCircuit::load(compiled_circuit_path)?;

// if both input and output are from files fail
Expand Down Expand Up @@ -1905,7 +1905,7 @@ pub(crate) async fn setup_test_evm_witness(
use crate::pfsys::ProofType;
pub(crate) async fn test_update_account_calls(
addr: H160Flag,
data: PathBuf,
data: String,
rpc_url: Option<String>,
) -> Result<String, EZKLError> {
use crate::eth::update_account_calls;
Expand Down
21 changes: 21 additions & 0 deletions src/graph/input.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,27 @@ impl GraphData {
}
}

/// Loads graph input data from a string, first seeing if it is a file path or JSON data
/// If it is a file path, it will load the data from the file
/// Otherwise, it will attempt to parse the string as JSON data
///
/// # Arguments
/// * `data` - String containing the input data
/// # Returns
/// A new GraphData instance containing the loaded data
pub fn from_str(data: &str) -> Result<Self, GraphError> {
let graph_input = serde_json::from_str(data);
match graph_input {
Ok(graph_input) => {
return Ok(graph_input);
}
Err(_) => {
let path = std::path::PathBuf::from(data);
GraphData::from_path(path)
}
}
}

/// Loads graph input data from a file
///
/// # Arguments
Expand Down
Loading
0