Enum ServeCommands 

pub enum ServeCommands {
    Start {
        model: String,
        engine: String,
        port: u16,
        replicas: u32,
        quantize: String,
    },
    List,
    Stop {
        model: String,
    },
    Logs {
        model: String,
    },
    Benchmark {
        url: String,
        qps: u32,
        duration: u32,
    },
}

Variants

Start

Start serving a model

Fields

model: String

Path to model or model name

engine: String

Inference engine (auto, vllm, trt, onnx)

port: u16

Port to serve on

replicas: u32

Number of GPU replicas

quantize: String

Quantization (none, int8, int4)

List

List running inference servers

Stop

Stop an inference server

Fields

model: String

Model name or port

Logs

Show server logs

Fields

model: String

Model name

Benchmark

Load test an inference endpoint

Fields

url: String

URL to benchmark

qps: u32

Queries per second

duration: u32

Duration in seconds

Trait Implementations

impl FromArgMatches for ServeCommands

fn from_arg_matches(__clap_arg_matches: &ArgMatches) -> Result<Self, Error>

Instantiate Self from [ArgMatches], parsing the arguments as needed.

fn from_arg_matches_mut( __clap_arg_matches: &mut ArgMatches, ) -> Result<Self, Error>

Instantiate Self from [ArgMatches], parsing the arguments as needed.

fn update_from_arg_matches( &mut self, __clap_arg_matches: &ArgMatches, ) -> Result<(), Error>

Assign values from ArgMatches to self.

fn update_from_arg_matches_mut<'b>( &mut self, __clap_arg_matches: &mut ArgMatches, ) -> Result<(), Error>

Assign values from ArgMatches to self.

impl Subcommand for ServeCommands

fn augment_subcommands<'b>(__clap_app: Command) -> Command

Append to [Command] so it can instantiate Self via [FromArgMatches::from_arg_matches_mut]

fn augment_subcommands_for_update<'b>(__clap_app: Command) -> Command

Append to [Command] so it can instantiate self via [FromArgMatches::update_from_arg_matches_mut]

fn has_subcommand(__clap_name: &str) -> bool

Test whether Self can parse a specific subcommand