D-Robotics Official Website
1.0.0
D-Robotics Official Website
1.0.0
English

Structure and Enumeration Types

Sampling Parameters

typedef struct common_params_sampling_s { int32_t top_k; // The top_k for sampling: a value ≤ 0 indicates using the entire vocabulary. float top_p; // The top_p sampling: a value of 1.0 means it is disabled. float min_p; // Minimum probability threshold: a value of 0.0 means it is disabled. float temp; // Temperature coefficient: greedy sampling is used when the value is ≤ 0.0. float typ_p; // The typical_p sampling: a value of 1.0 means it is disabled. int32_t min_keep; // Minimum number of tokens to retain. int32_t penalty_last_n; // Control the scope of history check: The repetition penalty only applies to the most recent n tokens. float penalty_repeat; // Repetition penalty coefficient: the larger the value, the less repetition there will be. float penalty_freq; // Frequency penalty coefficient: the larger the value, the less repetition there will be. float penalty_present; // Presence penalty coefficient: a penalty is applied once for any token that has appeared. } common_params_sampling_t;
  • Parameter settings for the model output sampling algorithm.

Model Types

typedef enum xlm_model_type_e { XLM_MODEL_TYPE_DEEPSEEK = 1, // 1: deepseek XLM_MODEL_TYPE_PI0 = 8, // 8: pi0 XLM_MODEL_TYPE_QWEN3 = 9, // 9: qwen3 XLM_MODEL_TYPE_WHISPER = 10, // 10: whisper XLM_MODEL_TYPE_VLM = 11 // 11: vlm } xlm_model_type;
  • Specify the model type.

Image Data Type

typedef enum xlm_image_data_type_e { XLM_IMAGE_DATA_TYPE_UINT8 = 0, // uint8 data type XLM_IMAGE_DATA_TYPE_FP16 = 1, // float16 data type XLM_IMAGE_DATA_TYPE_FP32 = 2 // float32 data type } xlm_image_data_type;
  • The data type used for storing image data.

VLA Parameters

typedef struct vla_params_s { const char *siglip_model_path; // vla visual模型路径 const char *paligemma_model_path; // vla vl模型路径 const char *action_model_path; // vla action模型路径 const char *norm_stats_path; // norm stats文件路径 }
  • Model and file configuration parameters for VLA (Pi0 only).

General Parameters

typedef struct xlm_common_params_s { const char *model_path; // Model file path (not supported yet) vlm_params_t vlm_param; // The vlm model parameter vla_params_t vla_param; // The vla model parameter (not support yet) const char *token_config_path; // Tokenizer configuration path const char *config_path; // Path to other configuration files bool k_cache_int8; // Whether to use k-int8 quantization xlm_model_type model_type; // Model type int32_t context_size; // Context Length int32_t max_img_cnt; // Maximum number of images per request common_params_sampling_t sampling; // Sampling Parameter char *prompt_file; // External prompt file name char *path_prompt_cache; // Prompt cache file path } xlm_common_params_t;
  • Configuration of model general parameters. The default parameters can be obtained via the xlm_create_default_param interface.
Warning

max_img_cnt represents the maximum number of input images per request. Note that the total number of tokens from the input images should not exceed the cache_len of the corresponding model.

Input Types

typedef enum xlm_input_type_e { XLM_INPUT_PROMPT = 0, // Plain text prompt input XLM_INPUT_TOKEN = 1, // Token ID input (not supported yet). XLM_INPUT_MULTI_MODAL = 2 // Multimodal Input XLM_INPUT_VLA = 3, // VLA Input XLM_INPUT_AUDIO = 4, // Audio Input } xlm_input_type;
  • The input type of the request.

Inference Backend Type

typedef enum xlm_infer_backend_e { XLM_INFER_BACKEND_ANY = 0, // Arbitrary Kernel XLM_INFER_BACKEND_BPU_ANY = 1, // Arbitrary BPU core XLM_INFER_BACKEND_BPU_0 = 2, // BPU core 0 XLM_INFER_BACKEND_BPU_1 = 3, // BPU core 1 XLM_INFER_BACKEND_BPU_2 = 4, // BPU core 2 XLM_INFER_BACKEND_BPU_3 = 5 // BPU core 3 } xlm_infer_backend;
  • Inference backend type setting, which supports the BPU core binding function.

Image Preprocessing type

typedef enum xlm_img_preprocess_type_e { XLM_IMG_PREPROCESS_DYNAMIC = 0, // Dynamic resolution (default). XLM_IMG_PREPROCESS_NONE = 1 // No preprocessing } xlm_img_preprocess_type;
  • Image preprocessing type setting. Currently, only dynamic resolution is supported.

Token Input Structure

typedef struct xlm_input_token_s { int32_t *tokens; // Token ID array int32_t tokens_size; // Token count } xlm_input_token_t;
  • The structure of the token input type.

Image Input Structure

typedef struct xlm_input_image_s { const char *image_path; // Image path (either one) const uint8_t *image_data; // Image data (either one) int32_t image_width; // Image width int32_t image_height; // Image height int32_t image_type; // Image format xlm_image_data_type image_data_type; // Image data type xlm_img_preprocess_type image_preprocess; // Preprocessing method } xlm_input_image_t;
  • The image structure in multimodal input.

Multimodal Input Structure

typedef struct xlm_input_multi_modal_s { const char *prompt; // Text prompt int32_t image_num; // Image count xlm_input_image_t *images; // Image array bool has_prompt; // Prompt available } xlm_input_multi_modal_t;

  • The structure in multimodal input.

    • Supports single prompt and single image.

    • Supports single prompt with multiple images.

Input Parameters

typedef struct xlm_input_vla_s { // tokens xlm_input_token_t tokens; // image int32_t image_num; // Number of images, supports multiple images per single prompt xlm_input_image_t *images; // state int32_t state_size; // State length double *state; // Raw state float *state_preproc; // Preprocessed state // reset bool reset; // Reset simulation } xlm_input_vla_t;
  • Input parameter struct for VLA.

Output Embed Struct

typedef struct xlm_output_embed_s { union { uint16_t *data_fp16; float *data_fp32; }; double *data_fp64; // embed int32_t *embed_shape; // embed dimensions int32_t embed_dim; // embed size } xlm_output_embed_t;
  • Struct for model output embeds.

VLA Output Parameters

typedef struct xlm_output_vla_s { int32_t embed_num; xlm_output_embed_t *embeds; xlm_output_embed_t *kvcache; } xlm_output_vla_t;
  • Output parameters for VLA.

Priority Type

typedef enum xlm_priority_type_e { XLM_PRIORITY_TYPE_NORMAL = 0, // Normal XLM_PRIORITY_TYPE_HIGH = 1, // High XLM_PRIORITY_TYPE_URGENT = 2 // Urgent } xlm_priority_type_t; typedef struct xlm_priority_s { xlm_priority_type_t type; // Priority type int32_t priority; // 0~253 Only valid when NORMAL, 0~253 } xlm_priority_t;

  • Model input request priority settings.

    • The preemption relationships are as follows:

      • XLM_PRIORITY_TYPE_URGENT --preemption--> XLM_PRIORITY_TYPE_HIGH --preemption--> XLM_PRIORITY_TYPE_NORMAL.

    • When the priority is both NORMAL, preemption will not occur. Instead, the execution order will be determined based on the value of priority.

      • A higher priority value indicates a higher priority. The value range is [0, 253].

Single Inference Request Structure

typedef struct xlm_lm_request_s { int32_t request_id; // Request id, corresponds one-to-one with the result returned xlm_input_type type; // Input type bool new_chat; // Whether it is a new conversation const char *prompt_json; // As input read by omni in json format bool need_partial_result; // Whether need to decode the result in segments union { const char *prompt; // Prompt xlm_input_token_t token; // Token id xlm_input_multi_modal_t multi_modal_requset; // Multimodal input xlm_input_vla_t vla_request; // The vla input }; const char *system_prompt; const char *chat_template; xlm_infer_backend infer_backend; // Inference backend xlm_priority_t priority; // Priority } xlm_lm_request_t;
  • Single inference request structure. Currently, only the InternVL model supports concurrent processing of up to 4 input requests.

Inference Input Structure

typedef struct xlm_input_s { int32_t request_num; // Request count xlm_lm_request_t *requests; // Request array } xlm_input_t;
  • Model inference input structure, which is passed as a parameter to the inference interface.

Audio Input Data Structure

typedef struct xlm_online_audio_s { const float *data; // Pointer to audio data size_t data_size; // Length of audio data } xlm_online_audio_t;
  • Input structure for providing audio data to the model.

Performance Data Structure

typedef struct xlm_model_performance_s { double vit_cost; // vit cost time in ms int64_t prefill_token_num; // prefill token count double prefill_tps; // prefill speed in tokens/s int64_t decode_token_num; // decode token count double decode_tps; // decode speed in tokens/s double ttft; // time to first token double tpot; // time per output token double end_to_end_cost; // end to end cost time } xlm_model_performance_t;
  • Model performance data structure. At the end of inference, it will return the performance data of this inference.

Inference Result Structure

typedef struct xlm_result_s { char *text; // Inference result text int32_t request_id; // Corresponding request ID xlm_model_performance_t performance; // Model performance data } xlm_result_t;
  • The structure for returning inference result.

Interface Status

typedef enum xlm_state_e { XLM_STATE_START = 0, // Start XLM_STATE_END = 1, // End XLM_STATE_RUNNING = 2, // Running XLM_STATE_ERROR = 3 // Error } xlm_state_t;
  • The current inference status of the model is returned along with the inference results.