triton-inference-server / dali_backend Goto Github PK

Traceback (most recent call last):
  File "grpc_client.py", line 209, in <module>
    main()
  File "grpc_client.py", line 182, in main
    results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs)
  File "/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/__init__.py", line 1086, in infer
    raise_error_grpc(rpc_error)
  File "/usr/local/lib/python3.8/dist-packages/tritonclient/grpc/__init__.py", line 61, in raise_error_grpc
    raise get_error_grpc(rpc_error) from None
tritonclient.utils.InferenceServerException: [StatusCode.UNAVAILABLE] Socket closed

platform: "ensemble"
max_batch_size: 16
input [
  {
    name: "IMAGE_RAW"
    data_type: TYPE_UINT8
    dims: [ -1 ]
  }
]
output [
  {
    name: "SCALE_RATIO" # ratio
    data_type: TYPE_FP32
    dims: [2]
  },
  {
    name: "NUM_DETECTIONS"
    data_type: TYPE_INT32
    dims: [ 1 ]
  },
  {
    name: "NMSED_SCORES"
    data_type: TYPE_FP32
    dims: [ 100 ]
  },
  {
    name: "NMSED_CLASSES"
    data_type: TYPE_FP32
    dims: [ 100 ]
  },
  {
    name: "SCALED_NMSED_BOXES"
    data_type: TYPE_FP32
    dims: [ 100, 4 ]
  }
]

ensemble_scheduling {
  step [
    {
      model_name: "dali_det_pre"
      model_version: -1
      input_map {
        key: "IMAGE_RAW"
        value: "IMAGE_RAW"
      }
      output_map {
        key: "DALI_OUTPUT_0"
        value: "NORM_IMG"
      }
      output_map {
        key: "DALI_OUTPUT_1"
        value: "SCALE_RATIO"
      }
    },
    {
      model_name: "face_det-ucs"
      model_version: -1
      input_map {
        key: "images"
        value: "NORM_IMG"
      }
      output_map {
        key: "num_detections"
        value: "NUM_DETECTIONS"
      }
      output_map {
        key: "nmsed_boxes"
        value: "NMSED_BOXES"
      }
      output_map {
        key: "nmsed_scores"
        value: "NMSED_SCORES"
      }
      output_map {
        key: "nmsed_classes"
        value: "NMSED_CLASSES"
      }
    },
    {
      model_name: "dali_det_post"
      model_version: -1
      input_map {
        key: "NMSED_BOXES"
        value: "NMSED_BOXES"
      }
      input_map {
        key: "SCALE_RATIO_INPUT"
        value: "SCALE_RATIO"
      }
      output_map {
        key: "SCALED_NMSED_BOXES_OUTPUT"
        value: "SCALED_NMSED_BOXES"
      }
    }
  ]
}

FLAGS = parse_args()

    triton_client = tritonclient.grpc.InferenceServerClient(url=FLAGS.url,
                                                            verbose=FLAGS.verbose)

    model_name = FLAGS.model_name
    model_version = -1

    print("Loading images")

    image_data = load_images(FLAGS.img_dir if FLAGS.img_dir is not None else FLAGS.img,
                             max_images=FLAGS.batch_size * FLAGS.n_iter)

    image_data = array_from_list(image_data)
    inputs = generate_inputs(FLAGS.input_names, image_data.shape, "UINT8")
    outputs = generate_outputs(FLAGS.output_names)

    # Initialize the data
    inputs[0].set_data_from_numpy(image_data)
    # Test with outputs
    results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs)
    print(results)

backend: "dali"
max_batch_size: 32
input [
  {
    name: "NMSED_BOXES"
    data_type: TYPE_FP32
    dims: [ 100, 4 ]
  },
  {
    name: "SCALE_RATIO_INPUT"
    data_type: TYPE_FP32
    dims: [ 2 ]
  }
]
output [
  {
    name: "SCALED_NMSED_BOXES_OUTPUT" 
    data_type: TYPE_FP32
    dims: [100, 4]
  }
]
dynamic_batching {
  preferred_batch_size: [ 4, 8, 16, 32 ]
  max_queue_delay_microseconds: 100
}

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import nvidia.dali as dali
import nvidia.dali.fn as fn
import nvidia.dali.types as types

pipe = dali.pipeline.Pipeline(batch_size=32, num_threads=8)
with pipe:
    nmsed_boxes = fn.external_source(device='gpu', name="NMSED_BOXES")
    scale_ratio = fn.external_source(device='gpu', name='SCALE_RATIO_INPUT')
   
    # Rescale BBOX
    ratio = fn.reductions.min(scale_ratio)
    nmsed_boxes /= ratio
    pipe.set_outputs(nmsed_boxes)

pipe.serialize(filename="1/model.dali")

for request in requests:
            # Get INPUT0
            in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0")
            # Get INPUT1
            in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1")

            out_0 = in_0.as_numpy() / np.min(in_1.as_numpy())

            # Create output tensors. You need pb_utils.Tensor
            # objects to create pb_utils.InferenceResponse.
            out_tensor_0 = pb_utils.Tensor("OUTPUT0",
                                           out_0.astype(output0_dtype))

            # Create InferenceResponse. You can set an error here in case
            # there was a problem with handling this inference request.
            # Below is an example of how you can set errors in inference
            # response:
            #
            # pb_utils.InferenceResponse(
            #    output_tensors=..., TritonError("An error occured"))
            inference_response = pb_utils.InferenceResponse(
                output_tensors=[out_tensor_0])
            responses.append(inference_response)

name: "python_det_post"
backend: "python"

input [
  {
    name: "INPUT0"
    data_type: TYPE_FP32
    dims: [ 100, 4 ]
    
  }
]
input [
  {
    name: "INPUT1"
    data_type: TYPE_FP32
    dims: [ 2 ]
    
  }
]
output [
  {
    name: "OUTPUT0"
    data_type: TYPE_FP32
    dims: [ 100, 4 ]
  }
]

instance_group [{ kind: KIND_CPU }]

I0206 08:36:48.496881 1 infer_response.cc:165] add response output: output: DALI_OUTPUT_0, type: FP32, shape: [64,3,512,320]
I0206 08:36:48.496914 1 pinned_memory_manager.cc:131] pinned memory allocation: size 125829120, addr 0x7f085e000090
I0206 08:36:48.496918 1 ensemble_scheduler.cc:509] Internal response allocation: DALI_OUTPUT_0, size 125829120, addr 0x7f085e000090, memory type 1, type id 139652737555264
I0206 08:36:48.496923 1 infer_response.cc:165] add response output: output: DALI_OUTPUT_1, type: INT64, shape: [64,3]
I0206 08:36:48.496926 1 pinned_memory_manager.cc:131] pinned memory allocation: size 1536, addr 0x7f08658000a0
I0206 08:36:48.496928 1 ensemble_scheduler.cc:509] Internal response allocation: DALI_OUTPUT_1, size 1536, addr 0x7f08658000a0, memory type 1, type id 139652737552464

I0207 07:39:24.490871 1 infer_response.cc:165] add response output: output: DALI_OUTPUT_0, type: FP32, shape: [64,3,512,320]
I0207 07:39:24.490905 1 pinned_memory_manager.cc:131] pinned memory allocation: size 125829120, addr 0x7f47fc000090
I0207 07:39:24.490909 1 ensemble_scheduler.cc:509] Internal response allocation: DALI_OUTPUT_0, size 125829120, addr 0x7f47fc000090, memory type 1, type id 0
I0207 07:39:24.490913 1 infer_response.cc:165] add response output: output: DALI_OUTPUT_1, type: INT64, shape: [64,3]
I0207 07:39:24.490917 1 ensemble_scheduler.cc:509] Internal response allocation: DALI_OUTPUT_1, size 1536, addr 0x7f47f8000000, memory type 2, type id 0
I0207 07:39:24.490986 1 ensemble_scheduler.cc:524] Internal response release: size 125829120, addr 0x7f47fc000090

-- Looking for pthread.h - found
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Failed
-- Looking for pthread_create in pthreads
-- Looking for pthread_create in pthreads - not found
-- Looking for pthread_create in pthread
-- Looking for pthread_create in pthread - found
-- Found Threads: TRUE  
-- DALI includes dir: /usr/local/lib/python3.6/dist-packages/nvidia/dali/include
-- DALI libs dir: /usr/local/lib/python3.6/dist-packages/nvidia/dali
-- DALI libs: dalidali_kernelsdali_operators
CMake Error at CMakeLists.txt:81 (add_subdirectory):
  The source directory

    /dali/extern/Catch2

  does not contain a CMakeLists.txt file.


-- Configuring incomplete, errors occurred!
See also "/dali/build_in_ci/CMakeFiles/CMakeOutput.log".
See also "/dali/build_in_ci/CMakeFiles/CMakeError.log".
The command '/bin/sh -c mkdir build_in_ci && cd build_in_ci &&     cmake                                                -D CMAKE_BUILD_TYPE=Release                        -D TRITON_DALI_SKIP_DOWNLOAD=ON ..                 -D CMAKE_INSTALL_PREFIX=/opt/tritonserver &&     make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install' returned a non-zero code: 1

I0714 12:40:59.798507 1 dali_model.h:92] Loading DALI pipeline from file models/dali/1/model.dali
E0714 12:40:59.798660 1 dali_model.cc:32] Failed to open serialized model file: models/dali/1/model.dali
I0714 12:40:59.798695 1 dali_backend.cc:169] TRITONBACKEND_ModelFinalize: delete model state
E0714 12:40:59.798719 1 model_repository_manager.cc:1348] failed to load 'dali' version 1: Unknown: DALI Backend error: Failed to open serialized model file: models/dali/1/model.dali

@dali.pipeline_def(batch_size=8, num_threads=min(mp.cpu_count(), 4), device_id=0)
def pipe():
    input0 = dali.fn.external_source(device="cpu", name="DALI_INPUT_0")
    sum = dali.fn.reductions.sum(input0)

    if (sum >= 0):
         string_result = "Positive"
    else:
         string_result = "Negative"

    string_result_e = np.char.encode(string_result, encoding='utf-8')

    output0 = dali.fn.external_source(source=string_result_e)

    return  output0;

name: "dali_simple_test"
backend: "dali"
max_batch_size: 8
input [
    {
        name: "DALI_INPUT_0"
        data_type: TYPE_INT8
        dims: [ -1 ]
    }
]
output [
    {
        name: "DALI_OUTPUT_0"
        data_type: TYPE_UINT8
        dims: [ -1 ]
    }
]

[/opt/dali/dali/operators/decoder/nvjpeg/nvjpeg_decoder_decoupled_api.h:546] [/opt/dali/dali/image/image_factory.cc:89] Assert on "CheckIsPNG(encoded_image, length) + CheckIsBMP(encoded_image, length) + CheckIsGIF(encoded_image, length) + CheckIsJPEG(encoded_image, length) + CheckIsTiff(encoded_image, length) + CheckIsPNM(encoded_image, length) + CheckIsJPEG2k(encoded_image, length) == 1" failed: Encoded image has ambiguous format

name: "dali"
backend: "dali"
max_batch_size: 256
input [
    {
     name: "DALI_INPUT_0"
     data_type: TYPE_STRING
     dims: [ -1 ]
    }
]

output [
    {
     name: "DALI_OUTPUT_0"
     data_type: TYPE_FP32
     dims: [ 224, 224, 3 ]
    }
]

import tritonclient.http as httpclient
import numpy as np


if __name__ == "__main__":
    img_bytes = open("sample.jpg", "rb").read()

    img_data = np.array([img_bytes], dtype=bytes)
    img_data = np.stack([img_data], axis=0)

    infer_input = httpclient.InferInput("DALI_INPUT_0", img_data.shape, "BYTES")
    infer_input.set_data_from_numpy(img_data, binary_data=True)
    triton_client = httpclient.InferenceServerClient(url="localhost:8000", verbose=True)

    result = triton_client.infer("dali", [infer_input])
    print(result)

(lldb) bt
* thread #15, name = 'tritonserver', stop reason = signal SIGSEGV: address access protected (fault address: 0x7ffe56096000)
    frame #0: 0x00007ffff69e96c3 libc.so.6`___lldb_unnamed_symbol1092$$libc.so.6 + 83
  * frame #1: 0x00007fff88368d15 libtriton_dali.so`void triton::backend::dali::detail::copy<triton::backend::dali::StorageCpu>(dst=0x00007ffa90003580, src=0x00007ffe56096000, count=2764800) at io_descriptor.h:78:9
    frame #2: 0x00007fff8836558e libtriton_dali.so`triton::backend::dali::IODescriptor<char, triton::backend::dali::StorageCpu, true>::append(this=0x00007ffa90003500, buffer="", size=2764800) at io_descriptor.h:160:33
    frame #3: 0x00007fff88357049 libtriton_dali.so`triton::backend::dali::detail::GenerateInputs(request=0x00007ffb640038d0) at dali_backend.h:148:24
    frame #4: 0x00007fff88357ac4 libtriton_dali.so`triton::backend::dali::detail::ProcessRequest(response=0x00007ffa90002fb0, request=0x00007ffb640038d0, executor=0x00007ffdf0006d90) at dali_backend.h:206:44
    frame #5: 0x00007fff8835b483 libtriton_dali.so`triton::backend::dali::TRITONBACKEND_ModelInstanceExecute(instance=0x00007ffdf0010340, reqs=0x00007ffa90000f30, request_count=1) at dali_backend.cc:423:71

def export_pipeline(
    input_name='DALI_INPUT_0',
    image_dim=[800, 800],
    batch_size=1,
    num_threads=1,
    device='cpu',
    padding=True,
    device_id=None,
    output_dir='./out/',
):

    pipeline = dali.pipeline.Pipeline(
        batch_size=batch_size,
        num_threads=num_threads,
        device_id=device_id
    )

    with pipeline:
        data = fn.external_source(name=input_name, device=device)

        images = fn.image_decoder(data, device=device)
        images = fn.resize(images, size=image_dim, mode="not_larger", max_size=image_dim)
        images = fn.pad(images, fill_value=0, shape=[image_dim[0], image_dim[1], 1])

        input_shape = fn.shapes(images)
        input_shape = fn.cast(input_shape, dtype=dali.types.FLOAT)

        images = fn.transpose(images, perm=[2, 0, 1])
        images = fn.cast(images, dtype=dali.types.FLOAT)

        shapes = fn.peek_image_shape(data)
        shapes = fn.cast(shapes, dtype=dali.types.FLOAT)

        out = [
            images,
            input_shape,
            shapes
        ]
        pipeline.set_outputs(*out)

@dali.pipeline_def(batch_size=64, num_threads=4, device_id=0)
def pipe():
    images = dali.fn.external_source(device="cpu", name="DALI_INPUT_0")
    images = dali.fn.decoders.image(images, device="mixed", output_type=types.RGB)
    images = dali.fn.resize(images, resize_x=224, resize_y=224, device='gpu')
    return dali.fn.crop_mirror_normalize(images,
                                           dtype=types.FLOAT16,
                                           output_layout="CHW",
                                           device='gpu',
                                           mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
                                           std=[0.229 * 255, 0.224 * 255, 0.225 * 255])

model_repository_dali_back/
├── dali
│   ├── 1
│   │   └── model.dali
│   └── config.pbtxt
├── ensemble_dali_vit
│   ├── 1
│   └── config.pbtxt
└── vit_base_trt
    ├── 1
    │   └── model.plan
    └── config.pbtxt

name: "dali"
backend: "dali"
max_batch_size: 64
input [
{
    name: "DALI_INPUT_0"
    data_type: TYPE_STRING
    dims: [ -1 ]
}
]
output [
{
    name: "DALI_OUTPUT_0"
    data_type: TYPE_FP16
    dims: [ 3, 224, 224 ]
}
]
parameters: [
  {
    key: "num_threads"
    value: { string_value: "4" }
  }
]
dynamic_batching {}

import numpy as np
import tritonclient.http as httpclient
from tritonclient.utils import triton_to_np_dtype

# Setup a connection with the Triton Inference Server. 
triton_client = httpclient.InferenceServerClient(url="localhost:8000")

input_name = "INPUT"
output_name = "OUTPUT"
model_name = "ensemble_dali_vit"
## READ IMAGE
img_bytes = open('test_img.png', "rb").read() ## Also try with .jpeg image
img_data = np.array([img_bytes], dtype=bytes)
transformed_img = np.stack([img_data], axis=0)

# Specify the names of the input and output layer(s) of our model.
test_input = httpclient.InferInput(input_name, transformed_img.shape, datatype="BYTES")
test_input.set_data_from_numpy(transformed_img, binary_data=True)
test_output = httpclient.InferRequestedOutput(output_name, binary_data=True)
# Querying the server
results = triton_client.infer(model_name=model_name, inputs=[test_input], outputs=[test_output])
test_output = results.as_numpy(output_name)
print(test_output)

import nvidia.dali as dali


@dali.pipeline_def(batch_size=64, num_threads=4, device_id=0)
def pipe():
    images = dali.fn.external_source(
        device="gpu", name="DALI_INPUT_0", dtype=dali.types.UINT8)
    images = dali.fn.color_space_conversion(
        images, image_type=dali.types.BGR, output_type=dali.types.RGB, device='gpu')
    images = dali.fn.resize(images, mode="not_larger",
                            resize_x=640, resize_y=384, device='gpu')
    images = dali.fn.crop(images, crop_w=640, crop_h=384, crop_pos_x=0, crop_pos_y=0,
                          fill_values=114, out_of_bounds_policy="pad", device='gpu')
    images = dali.fn.transpose(images, perm=[2, 0, 1], device='gpu')
    images = dali.fn.cast(images, dtype=dali.types.FLOAT, device='gpu')
    return images


pipe().serialize(filename="1/model.dali")

name: "preprocessbgr"
backend: "dali"
max_batch_size: 64 
input [
{
    name: "DALI_INPUT_0"
    data_type: TYPE_UINT8
    dims: [ -1, -1, 3 ]
}
]
 
output [
{
    name: "OUTPUT_0"
    data_type: TYPE_FP32
    dims: [ 3, 384, 640 ]
}
]
dynamic_batching { }

Signal (11) received.
 0# 0x00005572DF1FBBD9 in tritonserver
 1# 0x00007F6BFF552210 in /usr/lib/x86_64-linux-gnu/libc.so.6
 2# 0x00007F6BF5602D40 in /usr/local/cuda/compat/lib.real/libcuda.so.1
 3# 0x00007F6BF57383E3 in /usr/local/cuda/compat/lib.real/libcuda.so.1
 4# 0x00007F6BF585EB02 in /usr/local/cuda/compat/lib.real/libcuda.so.1
 5# 0x00007F6BF55BF2E3 in /usr/local/cuda/compat/lib.real/libcuda.so.1
 6# 0x00007F6BF55BFAC4 in /usr/local/cuda/compat/lib.real/libcuda.so.1
 7# 0x00007F6BF55C1BD5 in /usr/local/cuda/compat/lib.real/libcuda.so.1
 8# 0x00007F6BF562FAAE in /usr/local/cuda/compat/lib.real/libcuda.so.1
 9# 0x00007F6BC80CF0D9 in /opt/tritonserver/backends/dali/libtriton_dali.so
10# 0x00007F6BC809EFED in /opt/tritonserver/backends/dali/libtriton_dali.so
11# 0x00007F6BC80F3B65 in /opt/tritonserver/backends/dali/libtriton_dali.so
12# 0x00007F6BC8098130 in /opt/tritonserver/backends/dali/libtriton_dali.so
13# dali::ThreadPool::ThreadMain(int, int, bool) in /opt/tritonserver/backends/dali/dali/libdali.so
14# 0x00007F6ABDAE526F in /opt/tritonserver/backends/dali/dali/libdali.so
15# 0x00007F6BFFDBE609 in /usr/lib/x86_64-linux-gnu/libpthread.so.0
16# clone in /usr/lib/x86_64-linux-gnu/libc.so.6

instance_group [
    {
        count: 1
        kind: KIND_GPU
        gpus: [ 0 ]
    }
]

import nvidia.dali as dali
import nvidia.dali.types as types
from nvidia.dali.plugin.triton import autoserialize
import numpy as np


@autoserialize
@dali.pipeline_def(batch_size=3, num_threads=1, device_id=0)
def pipe():
    max_size = 896
    fill_value = 114.0/255

    images = dali.fn.external_source(device="gpu", name="images")
    images = dali.fn.crop_mirror_normalize(images, output_layout=types.NCHW, scale=1.0/255, dtype=types.FLOAT)
    images = dali.fn.resize(images, max_size=max_size, mode="not_larger")

    shapes = dali.fn.shapes(images, dtype=types.INT64)

    w = dali.fn.slice(shapes, 2, 1, axes=[0])
    h = dali.fn.slice(shapes, 1, 1, axes=[0])

    dx = dali.fn.cast((max_size - w) // 2, dtype=types.FLOAT)
    dy = dali.fn.cast((max_size - h) // 2, dtype=types.FLOAT)

    shift = dali.fn.stack(dx, dy, 0.0, axis=0)
    transform = dali.fn.cat(np.identity(3, dtype=np.float32), shift, axis=1)

    letterboxed = dali.fn.warp_affine(images, transform, size=[max_size, max_size], fill_value=fill_value, interp_type=types.INTERP_LINEAR)
    
    return letterboxed

E0716 06:43:17.929970 5269 logging.cc:43] coreReadArchive.cpp (32) - Serialization Error in verifyHeader: 0 (Magic tag does not match)
E0716 06:43:17.930068 5269 logging.cc:43] INVALID_STATE: std::exception
E0716 06:43:17.930074 5269 logging.cc:43] INVALID_CONFIG: Deserialize the cuda engine failed.
W0716 06:43:17.930082 5269 autofill.cc:225] Autofiller failed to detect the platform for retinaface_preprocess (verify contents of model directory or use --log-verbose=1 for more details)
W0716 06:43:17.930086 5269 autofill.cc:248] Proceeding with simple config for now
I0716 06:43:17.930418 5269 model_repository_manager.cc:810] loading: retinaface_preprocess:1
E0716 06:43:17.941151 5269 model_repository_manager.cc:986] failed to load 'retinaface_preprocess' version 1: Not found: unable to load backend library: /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block
ERROR: infer_trtis_server.cpp:1044 Triton: failed to load model retinaface_preprocess, triton_err_str:Invalid argument, err_msg:load failed for model 'retinaface_preprocess': version 1: Not found: unable to load backend library: /usr/lib/x86_64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block;

ERROR: infer_trtis_backend.cpp:45 failed to load model: retinaface_preprocess, nvinfer error:NVDSINFER_TRTIS_ERROR
ERROR: infer_trtis_backend.cpp:184 failed to initialize backend while ensuring model:retinaface_preprocess ready, nvinfer error:NVDSINFER_TRTIS_ERROR
0:00:09.727932979  5269      0x4ab4780 ERROR          nvinferserver gstnvinferserver.cpp:362:gst_nvinfer_server_logger:<primary-inference> nvinferserver[UID 5]: Error in createNNBackend() <infer_trtis_context.cpp:246> [UID = 5]: failed to initialize trtis backend for model:retinaface_preprocess, nvinfer error:NVDSINFER_TRTIS_ERROR
I0716 06:43:17.941406 5269 server.cc:280] Waiting for in-flight requests to complete.
I0716 06:43:17.941431 5269 server.cc:295] Timeout 30: Found 0 live models and 0 in-flight non-inference requests
0:00:09.728082338  5269      0x4ab4780 ERROR          nvinferserver gstnvinferserver.cpp:362:gst_nvinfer_server_logger:<primary-inference> nvinferserver[UID 5]: Error in initialize() <infer_base_context.cpp:81> [UID = 5]: create nn-backend failed, check config file settings, nvinfer error:NVDSINFER_TRTIS_ERROR
0:00:09.728095248  5269      0x4ab4780 WARN           nvinferserver gstnvinferserver_impl.cpp:439:start:<primary-inference> error: Failed to initialize InferTrtIsContext
0:00:09.728099778  5269      0x4ab4780 WARN           nvinferserver gstnvinferserver_impl.cpp:439:start:<primary-inference> error: Config file path: /data/deepstream-retinaface/dstest_ssd_nopostprocess.txt
0:00:09.728476338  5269      0x4ab4780 WARN           nvinferserver gstnvinferserver.cpp:460:gst_nvinfer_server_start:<primary-inference> error: gstnvinferserver_impl start failed
Error: gst-resource-error-quark: Failed to initialize InferTrtIsContext (1): gstnvinferserver_impl.cpp(439): start (): /GstPipeline:pipeline0/GstNvInferServer:primary-inference:

import nvidia.dali as dali
import nvidia.dali.types as types
import numpy as np
 
batch_size = 1
pipe = dali.pipeline.Pipeline(batch_size=batch_size, num_threads=1, device_id=0)

# Create the pipeline
with pipe:
    x = dali.fn.external_source(device="cpu", name="DALI_X_INPUT")
    y = dali.fn.external_source(device="cpu", name="DALI_Y_INPUT")
    pipe.set_outputs(x, y)

# Build the pipeline and save to file for testing in TRITON
pipe.build()
pipe.serialize(filename="./model_multi_input.dali")

# Input the test data
pipe.feed_input("DALI_X_INPUT", np.array([1200.0]*batch_size, dtype=np.float32))
pipe.feed_input("DALI_Y_INPUT", np.array([1920.0]*batch_size, dtype=np.float32))

# Run the pipeline
pipe_out = pipe.run()

# Check the output data
print(f"X Input: {np.array(pipe_out[0][0])}, Y Input: {np.array(pipe_out[1][0])}")
# Outputs: "X Input: 1200.0, Y Input: 1920.0"

name: "dali_multi_input"
backend: "dali"
max_batch_size: 1
default_model_filename: "model_multi_input.dali"
input [
  {
    name: "DALI_X_INPUT"
    data_type: TYPE_UINT8
    dims: [ -1 ]
  },
  {
    name: "DALI_Y_INPUT"
    data_type: TYPE_UINT8
    dims: [ -1 ]
  }
]

output [
  {
    name: "DALI_OUTPUT_X"
    data_type: TYPE_UINT8
    dims: [ -1 ]
  },
  {
    name: "DALI_OUTPUT_Y"
    data_type: TYPE_UINT8
    dims: [ -1 ]
  }
]

import argparse
import numpy as np
import os
from builtins import range
import sys

import tritonclient.grpc as grpcclient
import tritonclient.grpc.model_config_pb2 as model_config
import tritonclient.http as httpclient
from tritonclient.utils import triton_to_np_dtype
from tritonclient.utils import InferenceServerException

FLAGS = None


def parse_model_grpc(model_metadata, model_config):
    """
    Check the configuration of a model to make sure it meets the
    requirements for an image classification network (as expected by
    this client)
    """
    if len(model_metadata.inputs) != 2:
        raise Exception("expecting 2 input, got {}".format(len(model_metadata.inputs)))

    if len(model_config.input) != 2:
        raise Exception(
            "expecting 2 input in model configuration, got {}".format(len(model_config.input)))

    input_metadata = model_metadata.inputs
    output_metadata = model_metadata.outputs

    return (input_metadata, output_metadata, model_config.max_batch_size)


def parse_model_http(model_metadata, model_config):
    """
    Check the configuration of a model to make sure it meets the
    requirements for an image classification network (as expected by
    this client)
    """
    if len(model_metadata['inputs']) != 2:
        raise Exception("expecting 2 input, got {}".format(len(model_metadata['inputs'])))

    if len(model_config['input']) != 2:
        raise Exception(
            "expecting 2 input in model configuration, got {}".format(len(model_config['input'])))

    input_metadata = model_metadata['inputs']
    output_metadata = model_metadata['outputs']

    return (input_metadata, output_metadata, model_config['max_batch_size'])


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-v',
                        '--verbose',
                        action="store_true",
                        required=False,
                        default=True,
                        help='Enable verbose output')
    parser.add_argument('-m',
                        '--model-name',
                        type=str,
                        required=False,
                        default='dali_multi_input',
                        help='Name of model. Default is preprocess_inception_ensemble.')
    parser.add_argument('-u',
                        '--url',
                        type=str,
                        required=False,
                        default='localhost:7000',
                        help='Inference server URL. Default is localhost:8000.')
    parser.add_argument('-i',
                        '--protocol',
                        type=str,
                        required=False,
                        default='HTTP',
                        help='Protocol (HTTP/gRPC) used to ' +
                        'communicate with inference service. Default is HTTP.')
    FLAGS = parser.parse_args()

    protocol = FLAGS.protocol.lower()

    try:
        if protocol == "grpc":
            # Create gRPC client for communicating with the server
            triton_client = grpcclient.InferenceServerClient(url=FLAGS.url, verbose=FLAGS.verbose)
        else:
            # Create HTTP client for communicating with the server
            triton_client = httpclient.InferenceServerClient(url=FLAGS.url, verbose=FLAGS.verbose)
    except Exception as e:
        print("client creation failed: " + str(e))
        sys.exit(1)

    model_name = FLAGS.model_name

    # Make sure the model matches our requirements, and get some
    # properties of the model that we need for preprocessing
    try:
        model_metadata = triton_client.get_model_metadata(model_name=model_name)
    except InferenceServerException as e:
        print("failed to retrieve the metadata: " + str(e))
        sys.exit(1)

    try:
        model_config = triton_client.get_model_config(model_name=model_name)
    except InferenceServerException as e:
        print("failed to retrieve the config: " + str(e))
        sys.exit(1)

    if FLAGS.protocol.lower() == "grpc":
        input_metadata, output_metadata, batch_size = parse_model_grpc(model_metadata, model_config.config)
    else:
        input_metadata, output_metadata, batch_size = parse_model_http(model_metadata, model_config)

    in_x = np.stack([np.array([400]*batch_size, dtype=np.uint8)], axis=0)
    in_y = np.stack([np.array([640]*batch_size, dtype=np.uint8)], axis=0)

    # Set the input data
    inputs = []
    if FLAGS.protocol.lower() == "grpc":
        inputs.append(grpcclient.InferInput(input_metadata[0].name, in_x.shape, "UINT8"))
        inputs[0].set_data_from_numpy(in_x)
        inputs.append(grpcclient.InferInput(input_metadata[1].name, in_y.shape, "UINT8"))
        inputs[1].set_data_from_numpy(in_y)
    else:
        inputs.append(httpclient.InferInput(input_metadata[0]['name'], in_x.shape, "UINT8"))
        inputs[0].set_data_from_numpy(in_x)
        inputs.append(httpclient.InferInput(input_metadata[1]['name'], in_y.shape, "UINT8"))
        inputs[1].set_data_from_numpy(in_y)


    output_names = [ output.name if FLAGS.protocol.lower() == "grpc" else output['name'] for output in output_metadata ]

    outputs = []
    for output_name in output_names:
        if FLAGS.protocol.lower() == "grpc":
            outputs.append(grpcclient.InferRequestedOutput(output_name))
        else:
            outputs.append(httpclient.InferRequestedOutput(output_name))

#This is where the client fails due to the server crash
    # Send request
    result = triton_client.infer(model_name, inputs, outputs=outputs)

    output_x = result.as_numpy('DALI_OUTPUT_X')
    output_y = result.as_numpy('DALI_OUTPUT_Y')

    print("PASS")

**Exception has occurred: HTTPConnectionClosed**
connection closed.

**Exception has occurred: InferenceServerException**
[StatusCode.UNAVAILABLE] Socket closed

I1231 11:50:04.685638 1 grpc_server.cc:270] Process for ModelMetadata, rpc_ok=1, 0 step START
I1231 11:50:04.685677 1 grpc_server.cc:225] Ready for RPC 'ModelMetadata', 1
I1231 11:50:04.685684 1 model_repository_manager.cc:496] GetInferenceBackend() 'dali_multi_input' version -1
I1231 11:50:04.685690 1 model_repository_manager.cc:452] VersionStates() 'dali_multi_input'
I1231 11:50:04.685756 1 grpc_server.cc:270] Process for ModelMetadata, rpc_ok=1, 0 step COMPLETE
I1231 11:50:04.685763 1 grpc_server.cc:408] Done for ModelMetadata, 0
I1231 11:50:04.686291 1 grpc_server.cc:270] Process for ModelConfig, rpc_ok=1, 0 step START
I1231 11:50:04.686317 1 grpc_server.cc:225] Ready for RPC 'ModelConfig', 1
I1231 11:50:04.686323 1 model_repository_manager.cc:496] GetInferenceBackend() 'dali_multi_input' version -1
I1231 11:50:04.689018 1 grpc_server.cc:270] Process for ModelConfig, rpc_ok=1, 0 step COMPLETE
I1231 11:50:04.689028 1 grpc_server.cc:408] Done for ModelConfig, 0
I1231 11:50:04.690189 1 grpc_server.cc:3089] Process for ModelInferHandler, rpc_ok=1, 1 step START
I1231 11:50:04.690210 1 grpc_server.cc:3082] New request handler for ModelInferHandler, 4
I1231 11:50:04.690218 1 model_repository_manager.cc:496] GetInferenceBackend() 'dali_multi_input' version -1
I1231 11:50:04.690227 1 model_repository_manager.cc:496] GetInferenceBackend() 'dali_multi_input' version -1
I1231 11:50:04.690246 1 infer_request.cc:502] prepared: [0x0x7fd11c0035a0] request id: , model: dali_multi_input, requested version: -1, actual version: 1, flags: 0x0, correlation id: 0, batch size: 1, priority: 0, timeout (us): 0
original inputs:
[0x0x7fd11c003908] input: DALI_Y_INPUT, type: UINT8, original shape: [1,1], batch + shape: [1,1], shape: [1]
[0x0x7fd11c003828] input: DALI_X_INPUT, type: UINT8, original shape: [1,1], batch + shape: [1,1], shape: [1]
override inputs:
inputs:
[0x0x7fd11c003828] input: DALI_X_INPUT, type: UINT8, original shape: [1,1], batch + shape: [1,1], shape: [1]
[0x0x7fd11c003908] input: DALI_Y_INPUT, type: UINT8, original shape: [1,1], batch + shape: [1,1], shape: [1]
original requested outputs:
DALI_OUTPUT_X
DALI_OUTPUT_Y
requested outputs:
DALI_OUTPUT_X
DALI_OUTPUT_Y

1 dali backend instances
embedding_array: (11190, 512)
unmatch_num/total_num: 0/11190

1 dali backend instances
embedding_array: (11190, 512)
unmatch_num/total_num: 7246/11190

2 dali backend instances
embedding_array: (11190, 512)
unmatch_num/total_num: 4040/11190

4 dali backend instances
embedding_array: (11190, 512)
unmatch_num/total_num: 1628/11190

8 dali backend instances
embedding_array: (11190, 512)
unmatch_num/total_num: 64/11190

@pipeline_def(batch_size=max_batch_size, num_threads=4, device_id=0)
def pipe():
    # input = fn.external_source(source="cpu", name="input_8n", parallel=True)
    input = fn.external_source(device="cpu", name="input_8n", dtype=types.UINT8)#, parallel=True)
    imgG = fn.decoders.image(input, output_type=types.DALIImageType.GRAY) 

    images_32f = fn.cast(imgG, dtype=types.DALIDataType.FLOAT) / 255.0

    input_A = fn.normalize(images_32f, mean=0.5, stddev=1/0.225)
    input_A = fn.expand_dims(input_A, axes=(0))

    image_B = fn.resize(images_32f, size=[640, 360], image_type=types.DALIImageType.GRAY)
    B_norm_c0 = fn.normalize(image_B, mean=0.485, stddev=0.229)
    B_norm_c1 = fn.normalize(image_B, mean=0.456, stddev=0.224)
    B_norm_c2 = fn.normalize(image_B, mean=0.406, stddev=0.225)
    image_B2 = fn.stack( B_norm_c0, B_norm_c1, B_norm_c2 )

    return image_B2, input_A

input = fn.external_source(device="cpu", name="input_8n", dtype=types.UINT8)
imgG = fn.decoders.image(input, output_type=types.DALIImageType.GRAY, device="mixed") # HWC layout

NAVAILABLE: Unknown: DALI Backend error: [/opt/dali/dali/pipeline/pipeline.cc:276] Assert on "device_id_ != CPU_ONLY_DEVICE_ID || device == "cpu"" failed: Cannot add a mixed operator decoders__Image with a GPU output, device_id should not be CPU_ONLY_DEVICE_ID. |
Stacktrace (15 entries):
| [frame 0]: /opt/tritonserver/backends/dali/dali/
| [frame 4]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x3386b) [0x7fb95c2f686b]  
| [frame 5]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x2c635) [0x7fb95c2ef635]   
| [frame 6]: /opt/tritonserver/backends/dali/libtriton_dali.so(TRITONBACKEND_ModelInstanceInitialize+0x3f6) [0x7fb95c2ded26] 
| [frame 7]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x307dee) [0x7fba44fcedee] 
| [frame 8]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x3093b3) [0x7fba44fd03b3] 
| [frame 9]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x301067) [0x7fba44fc8067]      
| [frame 10]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x18a7ca) [0x7fba44e517ca] 
| [frame 11]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x1979b1) [0x7fba44e5e9b1]   
| [frame 12]: /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xd6de4) [0x7fba4481cde4]  
| [frame 13]: /usr/lib/x86_64-linux-gnu/libpthread.so.0(+0x9609) [0x7fba44c9a609] 
| [frame 14]: /usr/lib/x86_64-linux-gnu/libc.so.6(clone+0x43) [0x7fba4450a293]

import nvidia.dali as dali

 @dali.pipeline_def(batch_size=256, num_threads=4, device_id=0)
 def pipe():
     images = dali.fn.external_source(device="cpu", name="DALI_INPUT_0")
     images = dali.fn.image_decoder(images, device="mixed")
     images = dali.fn.resize(images, resize_x=224, resize_y=224)
     return images

 pipe().serialize(filename="/my/model/repository/path/dali/1/model.dali")

I0528 18:03:34.171215 1 dali_backend.cc:316] TRITONBACKEND_ModelInitialize: dali (version 1)
I0528 18:03:34.171221 1 dali_backend.cc:328] Repository location: /tmp/folderAQX4gi
I0528 18:03:34.171224 1 dali_backend.cc:339] backend state is 'backend state'
I0528 18:03:34.171954 1 dali_backend.cc:126] Loading DALI pipeline from file /tmp/folderAQX4gi/1/model.dali
I0528 18:03:34.171996 1 dali_backend.cc:79] model configuration:
{
    "name": "dali",
    "platform": "",
    "backend": "dali",
    "version_policy": {
        "latest": {
            "num_versions": 1
        }
    },
    "max_batch_size": 1,
    "input": [

I0528 18:03:34.211270 1 model_repository_manager.cc:1043] loading: model:1
I0528 18:03:35.426644 1 libtorch.cc:981] TRITONBACKEND_ModelInitialize: model (version 1)
I0528 18:03:35.427339 1 libtorch.cc:1022] TRITONBACKEND_ModelInstanceInitialize: model_0 (device 0)
I0528 18:03:38.321389 1 model_repository_manager.cc:1210] successfully loaded 'model' version 1

output: [
    {
        name: "processed_image"
        dims: [256, 256, 3]
        data_type: TYPE_FP32 
    },
    {
        name: "image_shape"
        dims: [3]
        data_type: TYPE_INT64
    }
]
# Image shape becomes processed_image and vice versa

Error when executing Mixed operator ImageDecoder encountered:
Error in thread 1: [/opt/dali/dali/operators/decoder/nvjpeg/nvjpeg_helper.h:167] [/opt/dali/dali/image/jpeg_mem.cc:441] Assert on "jpeg_finish_decompress(&cinfo)" failed

#8 222.9 [ 65%] Building CXX object Source/CMakeFiles/CMakeLib.dir/cmInstallGenerator.cxx.o
#8 245.6 g++: fatal error: Killed signal terminated program cc1plus
#8 245.6 compilation terminated.
#8 245.6 make[2]: *** [Source/CMakeFiles/CMakeLib.dir/build.make:1266: Source/CMakeFiles/CMakeLib.dir/cmGeneratorTarget.cxx.o] Error 1
#8 245.6 make[2]: *** Waiting for unfinished jobs....
#8 251.4 make[1]: *** [CMakeFiles/Makefile2:2205: Source/CMakeFiles/CMakeLib.dir/all] Error 2
#8 251.4 make: *** [Makefile:183: all] Error 2
------
executor failed running [/bin/sh -c CMAKE_VERSION=3.17 &&     CMAKE_BUILD=3.17.4 &&     wget -nv https://cmake.org/files/v${CMAKE_VERSION}/cmake-${CMAKE_BUILD}.tar.gz &&     tar -xf cmake-${CMAKE_BUILD}.tar.gz &&     cd cmake-${CMAKE_BUILD} &&     ./bootstrap --parallel=$(grep ^processor /proc/cpuinfo | wc -l) -- -DCMAKE_USE_OPENSSL=OFF &&     make -j"$(grep ^processor /proc/cpuinfo | wc -l)" install &&     rm -rf /cmake-${CMAKE_BUILD}]: exit code: 2

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import nvidia.dali as dali
import nvidia.dali.fn as fn
import nvidia.dali.types as types

pipe = dali.pipeline.Pipeline(batch_size=32, num_threads=8)
with pipe:
    expect_output_size = (120., 120.)
    images = fn.external_source(device='cpu', name="IMAGE_ARRAY", layout='HWC')
    images = images.gpu()
    raw_shapes = fn.shapes(images, dtype=types.UINT32)
    # images = fn.image_decoder(images, device="mixed", output_type=types.RGB)
    images = fn.resize(images, mode='not_larger', max_size=expect_output_size, size=expect_output_size)
    resized_shapes = fn.shapes(images, dtype=types.UINT32)
    ratio = fn.cast(resized_shapes / raw_shapes, dtype=types.FLOAT)
    images = fn.pad(images, axis_names="HW", align=expect_output_size)
    images = fn.crop_mirror_normalize(images, crop=expect_output_size, output_layout='CHW')
    pipe.set_outputs(images, ratio, raw_shapes)

pipe.serialize(filename="1/model.dali")

backend: "dali"
max_batch_size: 32
input [
  {
    name: "IMAGE_ARRAY"
    data_type: TYPE_UINT8
    format: FORMAT_NHWC
    dims: [ -1, -1, 3]
  }
]
output [
   {
    name: "DALI_OUTPUT_0" # image_normalized
    data_type: TYPE_FP32
    dims: [3, 120, 120]
  },
  {
    name: "DALI_OUTPUT_1" # ratio
    data_type: TYPE_FP32
    dims: [3]
  },
  {
    name: "DALI_OUTPUT_2" # origin_shape
    data_type:TYPE_UINT32
    dims: [3]
  }
]
dynamic_batching {
  preferred_batch_size: [ 4, 8, 16, 32 ]
  max_queue_delay_microseconds: 1000
}

@pipeline_def(batch_size=256, num_threads=4, device_id=0)
def pipeline():
    images = fn.external_source(device="cpu", name="DALI_INPUT_0", layout="HWC")
    ...

E1015 07:50:36.366937 1 dali_model_instance.cc:43] [/opt/dali/dali/pipeline/operator/builtin/external_source.h:567] Assert on "layout_ == batch.GetLayout()" failed: Expected data with layout: "HWC" and got: "".
Stacktrace (21 entries):
[frame 0]: /opt/tritonserver/backends/dali/dali/libdali.so(+0x869b2) [0x7efe0f3e89b2]
[frame 1]: /opt/tritonserver/backends/dali/dali/libdali.so(+0x1e8f98) [0x7efe0f54af98]
[frame 2]: /opt/tritonserver/backends/dali/dali/libdali.so(void dali::Pipeline::SetDataSourceHelper<dali::TensorList<dali::CPUBackend>, dali::CPUBackend>(std::string const&, dali::TensorList<dali::CPUBackend> const&, dali::OperatorBase*, dali::AccessOrder, dali::ExtSrcSettingMode)+0x94) [0x7efe0f5554a4]
[frame 3]: /opt/tritonserver/backends/dali/dali/libdali.so(void dali::Pipeline::SetExternalInputHelper<dali::TensorList<dali::CPUBackend> >(std::string const&, dali::TensorList<dali::CPUBackend> const&, dali::AccessOrder, dali::ExtSrcSettingMode)+0x107) [0x7efe0f55ab97]
[frame 4]: /opt/tritonserver/backends/dali/dali/libdali.so(daliSetExternalInputAsync+0xe8f) [0x7efe0f54705f]
[frame 5]: /opt/tritonserver/backends/dali/dali/libdali.so(daliSetExternalInput+0x1d) [0x7efe0f54764d]
[frame 6]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x3df29) [0x7efe684c5f29]
[frame 7]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x3df8c) [0x7efe684c5f8c]
[frame 8]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x3e202) [0x7efe684c6202]
[frame 9]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x3bbe0) [0x7efe684c3be0]
[frame 10]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x3becc) [0x7efe684c3ecc]
[frame 11]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x2ad05) [0x7efe684b2d05]
[frame 12]: /opt/tritonserver/backends/dali/libtriton_dali.so(+0x2b4a6) [0x7efe684b34a6]
[frame 13]: /opt/tritonserver/backends/dali/libtriton_dali.so(TRITONBACKEND_ModelInstanceExecute+0x190) [0x7efe684a0e00]
[frame 14]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x11290a) [0x7efe7585990a]
[frame 15]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x1130b7) [0x7efe7585a0b7]
[frame 16]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x1e5541) [0x7efe7592c541]
[frame 17]: /opt/tritonserver/bin/../lib/libtritonserver.so(+0x10d287) [0x7efe75854287]
[frame 18]: /usr/lib/x86_64-linux-gnu/libstdc++.so.6(+0xd6de4) [0x7efe75394de4]
[frame 19]: /usr/lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7efe7670d609]
[frame 20]: /usr/lib/x86_64-linux-gnu/libc.so.6(clone+0x43) [0x7efe7507f133]

@pipeline_def(batch_size=256, num_threads=4, device_id=0)
def pipeline():
    images = fn.external_source(device="cpu", name="DALI_INPUT_0")
    images = fn.reinterpret(images, layout="HWC")
    ...

def setup_dali(
    input_name='DALI_INPUT_0',
    image_dim=[800, 1600],
    batch_size=1,
    num_threads=4,
    device='cpu',
    device_id=0,
    output_dir='./out/',
):

    pipeline = dali.pipeline.Pipeline(
        batch_size=batch_size,
        num_threads=num_threads,
        device_id=device_id
    )

    with pipeline:
        data = fn.external_source(name=input_name, device="cpu")
        # image preprocess
        images = fn.image_decoder(data, device=device)
        images = fn.resize(images, size=image_dim, mode="not_larger", max_size=image_dim)
        images = fn.pad(images, fill_value=0, shape=[image_dim[0], image_dim[1], 1])
        images = fn.transpose(images, perm=[2, 0, 1])
        images = fn.cast(images, dtype=dali.types.FLOAT)
        # input shape
        input_shape = np.float32((image_dim[0], image_dim[1], 1))
        # original shape
        shapes = fn.peek_image_shape(data)
        shapes = fn.cast(shapes, dtype=dali.types.FLOAT)
        # gather outputs
        out = [
            images,
            input_shape,
            shapes
        ]
        pipeline.set_outputs(*out)

    os.makedirs(os.path.dirname(output_dir), exist_ok=True)
    pipeline.serialize(filename=os.path.join(output_dir, 'model.dali'))

from nvidia.dali.pipeline import Pipeline
import nvidia.dali.ops as ops
import nvidia.dali.types as types
from timeit import default_timer as timer

image_dir = "/test_imgs"

class HybridPipeline(Pipeline):
    def __init__(self, batch_size, num_threads, device_id):
        super(HybridPipeline, self).__init__(batch_size, num_threads, device_id, seed = 12)
        self.input = ops.FileReader(file_root=image_dir, random_shuffle=True, initial_fill=21)
        self.decode = ops.ImageDecoder(device='mixed', output_type=types.RGB)
        self.resize = ops.Resize(device='gpu',
                                 interp_type=types.INTERP_LINEAR,
                                 resize_x=512,
                                 resize_y=320)

    def define_graph(self):
        jpegs, labels = self.input()
        images = self.decode(jpegs)
        images = self.resize(images)
        # images are on the GPU
        return images


def speedtest(pipeclass, batch, n_threads):
    pipe = pipeclass(batch, n_threads, 0)
    pipe.build()
    # warmup
    for i in range(5):
        pipe.run()
    # test
    n_test = 200
    t_start = timer()
    for i in range(n_test):
        pipe.run()
    t = timer() - t_start
    print("Speed: {} imgs/s".format((n_test * batch)/t))


if __name__ == "__main__":
    test_batch_size = 64
    speedtest(HybridPipeline, test_batch_size, 4)

read 17365 files from 1 directories
Speed: 1757.3024810392662 imgs/s

import nvidia.dali as dali
import nvidia.dali.types as types
import argparse

INPUT_HEIGHT = 512
INPUT_WIDTH = 320

def main(filename):
    pipe = dali.pipeline.Pipeline(batch_size=64, num_threads=10, device_id=0)
    with pipe:
        images = dali.fn.external_source(device="cpu", name="DALI_INPUT_0")
        images = dali.fn.image_decoder(images, device="mixed", output_type=types.BGR)
        images = dali.fn.resize(images, size=(INPUT_HEIGHT, INPUT_WIDTH))
        pipe.set_outputs(images)
        pipe.serialize(filename=filename)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Serialize pipeline and save it to file")
    parser.add_argument('file_path', type=str, help='Path, where to save serialized pipeline')
    args = parser.parse_args()
    main(args.file_path)

./perf_client -v -i grpc -m dali_ctdet  --request-rate-range 50  -b 64 -p 5000 -u localhost:8001 --input-data /json_data/test_jpg.json

 Successfully read data for 1 stream/streams with 256 step/steps.
*** Measurement Settings ***
  Batch size: 64
  Measurement window: 5000 msec
  Using uniform distribution on request generation
  Using synchronous calls for inference
  Stabilizing using average latency

Request Rate: 50 inference requests per seconds
  Pass [1] throughput: 1100.8 infer/sec. Avg latency: 250086 usec (std 118550 usec)
  Pass [2] throughput: 1100.8 infer/sec. Avg latency: 233251 usec (std 10702 usec)
  Pass [3] throughput: 1100.8 infer/sec. Avg latency: 234106 usec (std 10974 usec)
  Client: 
    Request count: 86
    Delayed Request Count: 86
    Throughput: 1100.8 infer/sec
    Avg latency: 234106 usec (standard deviation 10974 usec)
    p50 latency: 233919 usec
    p90 latency: 250165 usec
    p95 latency: 253393 usec
    p99 latency: 260765 usec
    Avg gRPC time: 232563 usec (marshal 710 usec + response wait 231850 usec + unmarshal 3 usec)
  Server: 
    Inference count: 6592
    Execution count: 103
    Successful request count: 103
    Avg request latency: 188862 usec (overhead 4 usec + queue 143417 usec + compute input 6 usec + compute infer 38842 usec + compute output 6593 usec)

Inferences/Second vs. Client Average Batch Latency
Request Rate: 50, throughput: 1100.8 infer/sec, latency 234106 usec